From cfca4ff293a673015229193b4929d6f24910c254 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 09:44:44 -0400 Subject: [PATCH 01/87] Add atomic paired owner appends for snapshot recovery --- .config/nextest.toml | 2 +- crates/cyber-store/src/error.rs | 2 + crates/cyber-store/src/store.rs | 146 +++++++++ crates/cyber-store/tests/store.rs | 292 ++++++++++++++++++ .../implementation.md | 66 ++++ .../managed-recovery.md | 29 +- .../specs/storage-events/spec.md | 25 ++ .../complete-daily-coding-surfaces/tasks.md | 32 +- 8 files changed, 588 insertions(+), 6 deletions(-) create mode 100644 openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md diff --git a/.config/nextest.toml b/.config/nextest.toml index 472f44968..29cac8003 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -162,7 +162,7 @@ binary_id(cyber-account) | binary_id(cyber-snapshot) | binary_id(cyber-snapshot::file_copy_platform) | binary_id(cyber-snapshot::snapshots) -| binary_id(cyber-store) & (test(writer::tests::) | test(migrate::tests::messaging_) | test(migrate::tests::usage_origin_)) +| binary_id(cyber-store) & (test(writer::tests::) | test(store::paired_tests::) | test(migrate::tests::messaging_) | test(migrate::tests::usage_origin_)) | binary_id(cyber-store::store) | binary_id(cyber-tools) & (test(tools::bash::shell_tests) | test(plugin_host::) | test(browser::) | test(dev_server::) | test(permissions::engine::tests::message_send) | test(permissions::engine::tests::remote_attach) | test(remote_skills) | test(lsp::) | test(mcp::stdio::tests) | test(mcp::roots::tests) | test(mcp::elicitation::tests) | test(mcp::sampling::tests) | test(tools::process::tests) | test(tools::tool_search::tests) | test(mcp::pool::tests) | test(mcp::launch::tests) | test(mcp::http::tests) | test(mcp::oauth) | test(hook_commands::launch::tests) | test(sandboxing::tests) | test(permissions::filesystem) | test(tools::background::tests) | test(tools::monitor::) | test(tools::deliver::tests) | test(tools::powershell::tests) | test(permissions::dry_run) | test(sandboxing::tests::refusal_lines) | test(permissions::engine::tests::sandbox_boundary_needs_an_explicit_user_allow) | test(permissions::engine::tests::workflow_run) | test(permissions::taint::) | test(untrusted::)) | binary_id(cyber-tools::agent_context) diff --git a/crates/cyber-store/src/error.rs b/crates/cyber-store/src/error.rs index 2ed3fd5c6..287ee2de0 100644 --- a/crates/cyber-store/src/error.rs +++ b/crates/cyber-store/src/error.rs @@ -30,6 +30,8 @@ pub enum StoreError { Projector { kind: String, reason: String }, #[error("InvalidLimitError: limit must be between 1 and 500, got {0}")] InvalidLimit(u32), + #[error("invalid paired append: {0}")] + InvalidAppend(String), #[error("the live database cannot be on a network filesystem ({0}); use a local disk")] NetworkFilesystem(String), #[error("another cyber server holds the writer lock {}", .0.display())] diff --git a/crates/cyber-store/src/store.rs b/crates/cyber-store/src/store.rs index 5437b52f7..d2b825975 100644 --- a/crates/cyber-store/src/store.rs +++ b/crates/cyber-store/src/store.rs @@ -212,6 +212,56 @@ impl Store { Ok((stored, result)) } + /// Decide and append to exactly two distinct owners in one writer transaction. + /// Used when a recovery receipt and its native acknowledgement must commit together. + /// Each owner supplies its exact latest sequence (`-1` for a new owner). Decision + /// writes, validation, both appends and projectors roll back together on failure; + /// commit notifications are published for each changed owner only after commit. + /// + /// Owner IDs are nonempty and at most 256 bytes. The pair may contain at most 64 + /// events and 16 MiB of encoded event data. The callback must not perform external + /// effects. A commit error may mean both owners committed: reconcile by durable + /// request identity after recovery, never redispatch filesystem effects. + pub fn append_pair_checked( + &self, + owners: [(String, i64); 2], + decide: F, + ) -> Result<([Vec; 2], T), StoreError> + where + T: Send + 'static, + F: FnOnce(&rusqlite::Transaction<'_>) -> Result<([Vec; 2], T), StoreError> + + Send + + 'static, + { + if owners[0].0 == owners[1].0 + || owners + .iter() + .any(|(id, seq)| id.is_empty() || id.len() > 256 || *seq < -1) + { + return Err(StoreError::InvalidAppend( + "expected two distinct bounded owner IDs and sequences >= -1".into(), + )); + } + let registry = Arc::clone(&self.registry); + let (stored, result) = self.transaction(move |tx| { + let (events, result) = decide(tx)?; + validate_pair(&events)?; + let mut stored = [vec![], vec![]]; + for (index, ((owner, sequence), events)) in owners.into_iter().zip(events).enumerate() { + for event in &events { + registry.check(event)?; + } + stored[index] = + append_in_tx(tx, ®istry, &owner, Expected::Seq(sequence), events)?; + } + Ok((stored, result)) + })?; + for events in &stored { + self.notify(events); + } + Ok((stored, result)) + } + /// Events of `aggregate_id` with `seq > after` (use `-1` for all), upcast to current versions. pub fn read_events( &self, @@ -352,6 +402,35 @@ impl Store { } } +fn validate_pair(events: &[Vec; 2]) -> Result<(), StoreError> { + if events.iter().map(Vec::len).sum::() > 64 { + return Err(StoreError::InvalidAppend("more than 64 events".into())); + } + let mut bytes = EventBudget(0); + for event in events.iter().flatten() { + serde_json::to_writer(&mut bytes, event) + .map_err(|_| StoreError::InvalidAppend("more than 16 MiB of events".into()))?; + } + Ok(()) +} + +/// Count the exact encoding without allocating an unbounded serialized copy. +struct EventBudget(usize); + +impl std::io::Write for EventBudget { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > (16 * 1024 * 1024) - self.0 { + return Err(std::io::Error::other("paired event byte limit")); + } + self.0 += bytes.len(); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + fn upcast(registry: &EventRegistry, row: RawEvent) -> Result { let data: Value = serde_json::from_str(&row.data).map_err(|e| StoreError::CorruptEvent { id: row.id.clone(), @@ -509,3 +588,70 @@ fn open_reader(path: &Path) -> Result { conn.execute_batch("PRAGMA query_only = ON; PRAGMA cache_size = -64000;")?; Ok(conn) } + +#[cfg(test)] +mod paired_tests { + use super::*; + + #[test] + fn paired_lost_commit_reply_retains_both_owners_without_notifications_or_replay() { + fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("injected paired commit reply loss".into()), + )) + } + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("paired.db"); + let mut registry = EventRegistry::default(); + registry.register("test.owner.changed.1").unwrap(); + let store = Store::open(StoreOptions::new( + DatabaseLocation::File(path.clone()), + registry.clone(), + )) + .unwrap(); + store + .transaction(|tx| { + tx.wal_hook(Some(lost_reply)); + Ok(()) + }) + .unwrap(); + let notifications = store.subscribe(); + let owners = [("native".to_owned(), -1), ("receipt".to_owned(), -1)]; + let events = || { + [ + vec![NewEvent::new( + "test.owner.changed.1", + serde_json::json!({"request": "same"}), + )], + vec![NewEvent::new( + "test.owner.changed.1", + serde_json::json!({"request": "same"}), + )], + ] + }; + let result = store.append_pair_checked(owners.clone(), move |_| Ok((events(), ()))); + assert!(matches!(result, Err(StoreError::Unavailable(_)))); + assert!(store.has_failed()); + assert!(notifications.try_recv().is_err()); + // Read through the independent file connection, not a cached writer result. + assert_eq!(store.aggregate_seq("native").unwrap(), Some(0)); + assert_eq!(store.aggregate_seq("receipt").unwrap(), Some(0)); + assert!(matches!( + store.append_pair_checked(owners.clone(), move |_| Ok((events(), ()))), + Err(StoreError::Unavailable(_)) + )); + drop(store); + let reopened = + Store::open(StoreOptions::new(DatabaseLocation::File(path), registry)).unwrap(); + assert!(matches!( + reopened.append_pair_checked(owners, move |_| Ok((events(), ()))), + Err(StoreError::Concurrency { .. }) + )); + for owner in ["native", "receipt"] { + let page = reopened.read_events(owner, -1, 10).unwrap(); + assert_eq!(page.events.len(), 1); + assert_eq!(page.events[0].data["request"], "same"); + } + } +} diff --git a/crates/cyber-store/tests/store.rs b/crates/cyber-store/tests/store.rs index f05d0ef99..ccb9a3ff7 100644 --- a/crates/cyber-store/tests/store.rs +++ b/crates/cyber-store/tests/store.rs @@ -437,6 +437,298 @@ fn checked_append_rolls_back_decision_mutations_when_events_are_invalid() { assert_eq!(stored_value, 2); } +fn pair_owners() -> [(String, i64); 2] { + [ + ("native_worker".into(), -1), + ("resolution_receipt".into(), -1), + ] +} + +#[test] +fn paired_append_commits_both_owners_before_notifying_and_survives_reopen() { + let dir = tempfile::tempdir().unwrap(); + let store = open(dir.path(), registry()); + let notifications = store.subscribe(); + let (events, result) = store + .append_pair_checked(pair_owners(), |tx| { + let count: i64 = tx.query_row("SELECT count(*) FROM event", [], |r| r.get(0))?; + assert_eq!(count, 0); + Ok(([vec![event(1), event(2)], vec![event(3)]], "acknowledged")) + }) + .unwrap(); + assert_eq!(result, "acknowledged"); + assert_eq!(events[0].iter().map(|e| e.seq).collect::>(), [0, 1]); + assert_eq!(events[1][0].seq, 0); + for (owner, sequence) in [("native_worker", 1), ("resolution_receipt", 0)] { + let notification = notifications.try_recv().unwrap(); + assert_eq!(notification.aggregate_id, owner); + assert_eq!(notification.seq, sequence); + // Even the first owner's notification exposes the complete committed pair. + assert_eq!(store.aggregate_seq("native_worker").unwrap(), Some(1)); + assert_eq!(store.aggregate_seq("resolution_receipt").unwrap(), Some(0)); + } + assert!(notifications.try_recv().is_err()); + drop(store); + let reopened = open(dir.path(), registry()); + assert_eq!( + reopened + .read_events("native_worker", -1, 10) + .unwrap() + .events + .len(), + 2 + ); + assert_eq!( + reopened + .read_events("resolution_receipt", -1, 10) + .unwrap() + .events + .len(), + 1 + ); +} + +#[test] +fn paired_append_late_failure_rolls_back_decision_events_sequences_and_projections() { + for failure in ["sequence", "validation", "projector"] { + assert_paired_failure_rolls_back(failure); + } +} + +fn assert_paired_failure_rolls_back(failure: &'static str) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let dir = tempfile::tempdir().unwrap(); + let mut registry = registry(); + registry + .register_with("test.invalid.data.1", |_| Err("refused payload".into())) + .unwrap(); + let projected = Arc::new(AtomicUsize::new(0)); + let seen = projected.clone(); + registry.projector(move |tx, event| { + seen.fetch_add(1, Ordering::SeqCst); + tx.execute( + "INSERT INTO paired_projection VALUES (?1)", + [&event.aggregate_id], + ) + .map_err(|error| error.to_string())?; + if failure == "projector" && event.aggregate_id == "resolution_receipt" { + return Err("refused resolution projection".into()); + } + Ok(()) + }); + let store = open(dir.path(), registry); + store.transaction(|tx| { + tx.execute_batch("CREATE TABLE paired_projection(owner TEXT); CREATE TABLE paired_decision(value INTEGER)")?; + Ok(()) + }).unwrap(); + let notifications = store.subscribe(); + let mut owners = pair_owners(); + if failure == "sequence" { + owners[1].1 = 0; + } + let result = store.append_pair_checked(owners, move |tx| { + tx.execute("INSERT INTO paired_decision VALUES (1)", [])?; + let second = if failure == "validation" { + NewEvent::new("test.invalid.data.1", json!({})) + } else { + event(2) + }; + Ok(([vec![event(1)], vec![second]], ())) + }); + match failure { + "sequence" => assert!(matches!(result, Err(StoreError::Concurrency { .. }))), + "validation" => assert!(matches!(result, Err(StoreError::InvalidEvent { .. }))), + _ => assert!(matches!(result, Err(StoreError::Projector { .. }))), + } + assert!( + projected.load(Ordering::SeqCst) >= 1, + "first append must actually reach its projector" + ); + assert_eq!(store.aggregate_seq("native_worker").unwrap(), None); + assert_eq!(store.aggregate_seq("resolution_receipt").unwrap(), None); + let counts: (i64, i64, i64) = store.read(|db| Ok(db.query_row( + "SELECT (SELECT count(*) FROM event), (SELECT count(*) FROM paired_projection), (SELECT count(*) FROM paired_decision)", + [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + )?)).unwrap(); + assert_eq!(counts, (0, 0, 0)); + assert!(notifications.try_recv().is_err()); + assert!( + !store.has_failed(), + "a rejected decision must not disable the writer" + ); +} + +#[test] +fn paired_append_rejected_commit_has_no_partial_receipt_or_notification() { + let dir = tempfile::tempdir().unwrap(); + let store = open(dir.path(), registry()); + store.transaction(|tx| { + tx.execute_batch("CREATE TABLE paired_parent(id INTEGER PRIMARY KEY); CREATE TABLE paired_child(parent_id INTEGER REFERENCES paired_parent(id) DEFERRABLE INITIALLY DEFERRED)")?; + Ok(()) + }).unwrap(); + let notifications = store.subscribe(); + let result = store.append_pair_checked(pair_owners(), |tx| { + tx.execute("INSERT INTO paired_child VALUES (1)", [])?; + Ok(([vec![event(1)], vec![event(2)]], ())) + }); + assert!(matches!(result, Err(StoreError::Sqlite(_)))); + for (owner, _) in pair_owners() { + assert_eq!(store.aggregate_seq(&owner).unwrap(), None); + } + assert!(notifications.try_recv().is_err()); + assert!(!store.has_failed()); + store + .append_pair_checked(pair_owners(), |_| { + Ok(([vec![event(1)], vec![event(2)]], ())) + }) + .unwrap(); + assert_eq!(store.aggregate_seq("native_worker").unwrap(), Some(0)); + assert_eq!(store.aggregate_seq("resolution_receipt").unwrap(), Some(0)); +} + +#[test] +fn paired_append_rejects_invalid_owners_without_running_the_decision() { + let dir = tempfile::tempdir().unwrap(); + let store = open(dir.path(), registry()); + for owners in [ + [("same".into(), -1), ("same".into(), -1)], + [(String::new(), -1), ("receipt".into(), -1)], + [("x".repeat(257), -1), ("receipt".into(), -1)], + [("native".into(), -2), ("receipt".into(), -1)], + ] { + let result = + store.append_pair_checked::<(), _>(owners, |_| panic!("invalid owner admitted")); + assert!(matches!(result, Err(StoreError::InvalidAppend(_)))); + } + assert!(!store.has_failed()); +} + +#[test] +fn paired_append_bounds_roll_back_decision_writes_without_notifications() { + let dir = tempfile::tempdir().unwrap(); + let store = open(dir.path(), registry()); + let notifications = store.subscribe(); + for events in [ + [(0..65).map(event).collect(), vec![]], + [ + vec![NewEvent::new( + KIND, + json!({"payload": "x".repeat(16 * 1024 * 1024)}), + )], + vec![], + ], + [ + vec![NewEvent::new( + KIND, + json!({"payload": "x".repeat(8 * 1024 * 1024)}), + )], + vec![NewEvent::new( + KIND, + json!({"payload": "x".repeat(8 * 1024 * 1024)}), + )], + ], + ] { + let result = store.append_pair_checked(pair_owners(), move |tx| { + tx.execute_batch("CREATE TABLE rejected_pair(value INTEGER)")?; + Ok((events, ())) + }); + assert!(matches!(result, Err(StoreError::InvalidAppend(_)))); + let count: i64 = store + .read(|db| { + Ok(db.query_row( + "SELECT count(*) FROM sqlite_master WHERE name='rejected_pair'", + [], + |r| r.get(0), + )?) + }) + .unwrap(); + assert_eq!(count, 0); + assert_eq!(store.aggregate_seq("native_worker").unwrap(), None); + assert!(notifications.try_recv().is_err()); + assert!(!store.has_failed()); + } +} + +#[test] +fn paired_append_empty_retry_still_checks_both_owners_and_does_not_notify() { + let dir = tempfile::tempdir().unwrap(); + let store = open(dir.path(), registry()); + let owners = [("n".repeat(256), -1), ("r".repeat(256), -1)]; + store + .append_pair_checked(owners.clone(), |_| { + Ok(( + [(0..32).map(event).collect(), (32..64).map(event).collect()], + (), + )) + }) + .unwrap(); + let notifications = store.subscribe(); + let owned = [(owners[0].0.clone(), 31), (owners[1].0.clone(), 31)]; + let (_, result) = store + .append_pair_checked(owned.clone(), |_| { + Ok(([vec![], vec![]], "retained receipt")) + }) + .unwrap(); + assert_eq!(result, "retained receipt"); + assert!(notifications.try_recv().is_err()); + let stale = [(owned[0].0.clone(), 31), (owned[1].0.clone(), 30)]; + assert!(matches!( + store.append_pair_checked(stale, |_| Ok(([vec![], vec![]], ()))), + Err(StoreError::Concurrency { .. }) + )); + let error = store.append_pair_checked::<(), _>(owned, |tx| { + tx.execute_batch("CREATE TABLE refused_decision(value INTEGER)")?; + Err(StoreError::InvalidAppend("decision refused".into())) + }); + assert!(matches!(error, Err(StoreError::InvalidAppend(_)))); + let count: i64 = store + .read(|db| { + Ok(db.query_row( + "SELECT count(*) FROM sqlite_master WHERE name='refused_decision'", + [], + |r| r.get(0), + )?) + }) + .unwrap(); + assert_eq!(count, 0); + assert!(notifications.try_recv().is_err()); + assert!(!store.has_failed()); +} + +#[test] +fn paired_append_concurrent_attempts_cannot_duplicate_either_owner() { + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new(open(dir.path(), registry())); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let workers: Vec<_> = (0..2) + .map(|_| { + let store = store.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + store.append_pair_checked(pair_owners(), |_| { + Ok(([vec![event(1)], vec![event(2)]], ())) + }) + }) + }) + .collect(); + let results: Vec<_> = workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect(); + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|r| matches!(r, Err(StoreError::Concurrency { .. }))) + .count(), + 1 + ); + for (owner, _) in pair_owners() { + assert_eq!(store.read_events(&owner, -1, 10).unwrap().events.len(), 1); + } +} + /// A projection rebuilt from its events: the tables are cleared and only events of the /// prefix are re-projected, upcast and in commit order; a failing projector leaves the /// previous rows in place. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 64f52358f..df51717fe 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -337,3 +337,69 @@ identity checks and implicit omission authority. New Git refs intentionally reta objects until a safe reclamation protocol is implemented. The managed recovery design is awaiting owner review and the remaining rules/context, structured output and JSONL are not claimed complete by this increment. + +## Managed recovery continuation baseline, 2026-11-06 + +Fast-forwarded the clean primary checkout from `b41f11f1` to current main +`db9e171e`. No open repository PRs were returned by the live PR inventory. +Other worktrees were inventoried and left untouched. The recovery execution +breakdown M1–M8 in tasks.md maps the existing protocol to actual worker, snapshot +and Store interfaces without implementing or approving it. At that point task 3.0 +remained open and the protocol was submitted for user review; the subsequent +decision and implementation are recorded below. + +On unchanged Rust sources at `db9e171e`, `cargo test --locked -p cyber-snapshot` +passed two storage unit tests, 17 file-copy cases and 17 Git snapshot cases on +macOS. The platform-specific integration binary and doc tests each selected zero +tests; they are not native Linux/Windows evidence. No external provider was used. +Strict OpenSpec 1.13.2 validation passed all 117 items, spec lint reported zero +errors and 25 warnings, and inventory reported 53 capabilities/1030 requirements +with no ROADMAP diff. Diff whitespace checks passed. The global `openspec` +command was unavailable; validation used `npx --yes @fission-ai/openspec@1.13.2`. +These initial baseline checks excluded workspace-wide Clippy, the SDK suite, +real-client acceptance and new platform CI. Managed recovery and the remaining +daily-coding/plugin scope are still incomplete. + +## Approved recovery protocol and atomic Store prerequisite + +The user approved the concrete protocol on 2026-11-06: "Approve this protocol +and implement." Tasks 3.0/M1 and managed-recovery.md now record that decision. +The independent Linux plugin support, Setup and DirectoryAdded gates remain +unchanged. The existing recovery protocol is the implementation scope. + +The first implementation prerequisite adds `Store::append_pair_checked` for the +native activity and resolution receipt owners. It requires two distinct bounded +IDs and exact sequences; decisions, events, sequence updates and projectors share +one writer transaction. A rejected second owner or COMMIT rolls back all changes. +Each changed owner is notified only after acknowledged commit. An actual WAL-hook +error after COMMIT proves both owners remain durable while notifications stay +silent and the writer is fenced; reopening permits inspecting the original records +and refuses the stale repeated append. This is Store evidence, not native worker +acknowledgement or a complete recovery retry implementation. + +The unchanged-main Store baseline passed 49 tests. The updated Store suite passed +57 tests (12 unit, one process-kill, three MCP, 15 reservation and 26 Store cases), +including eight new paired-transaction cases. Coverage includes late sequence, +validation and projector failures, rolled-back decision SQL, deferred COMMIT +rejection, concurrency, restart, invalid owners, empty retries and cumulative +encoded-byte/event limits. Package all-target Clippy with warnings denied passed. +Workspace all-target Clippy with warnings denied also passed. Strict OpenSpec +validation passed all 117 items after the storage-events delta; spec lint retained +zero errors/25 warnings, inventory left ROADMAP unchanged, and formatting/diff +checks passed. No SDK/client generation is needed for this internal Store API. +All results here are local macOS results; Linux/Windows CI is not yet run. The +Windows selection explicitly includes the new Store lost-COMMIT unit regression; +its Store integration binary was already selected. + +The code-review skill pass identified that size validation should not allocate an +unbounded serialized event copy; it now counts exact JSON encoding into a limited +sink. The pass also added cumulative cross-owner and empty-retry coverage and +explicit Windows selection. Clippy initially rejected one test's complexity +(16/15); extracting the shared scenario assertion resolved it without weakening +its assertions. The cognitive-complexity skill does not support Rust; the +repository's Clippy check supplies the applicable Rust gate. + +No HTTP/SDK/CLI contract or database schema is changed yet. Exact snapshot-worker +origin, worker lock retention, new journals, owner recovery operations and safe +reclamation remain required before M2–M8 or the parent recovery tasks can close. +No recovery success or native acknowledgement is inferred from this prerequisite. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 9a982f27c..ede9ca85c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -1,9 +1,12 @@ # Managed snapshot recovery: ownership and resolution design -Status: proposed implementation detail for review. The user approved implementing -managed keep/rollback, native acknowledgement and cleanup on 2026-11-05. The original -plan and #110 deliberately did not select the ownership/migration protocol below. -This document makes that protocol reviewable before changing recovery admission. +Status: approved for implementation on 2026-11-06. The user reviewed the concrete +protocol below and replied, "Approve this protocol and implement." This includes +fingerprint-confirmed keep/rollback, exact worker-bound acknowledgement, retry +lookup without replay, legacy unknown fences and cleanup after durable +acknowledgement/reference checks. The earlier 2026-11-05 feature request alone +did not select this protocol. Approval does not establish implementation or +acceptance; the native/platform and complete recovery tests remain required. ## Owner-visible contract @@ -70,6 +73,24 @@ identities are rechecked before every filesystem effect and the final commit. proof. If journal acknowledgement fails, preserve payloads and permit only acknowledgement reconciliation, not another rollback. +## Store transaction prerequisite + +The native activity and owner resolution receipt are two separate durable +aggregates. The Store's paired checked append accepts exactly two distinct owner +IDs (1–256 bytes), each with an exact expected sequence (`-1` for creation), +and at most 64 events/16 MiB of encoded events across the pair. The writer runs +the decision, both event batches and all projectors in one IMMEDIATE transaction. +Any decision, validation, CAS, projection or rejected-commit error rolls back +both owners and decision writes. Each changed owner is notified only after +observed COMMIT success. Encoded-size validation counts into a bounded sink rather +than allocating a serialized copy. The callback cannot perform filesystem or +other external effects. + +An error after SQLite committed must fence new writes and withhold notifications; +the owner reconciles the original durable request/receipt after recovery instead +of repeating filesystem changes. This Store primitive alone does not establish +native proof, admit a recovery operation or implement a resolution receipt. + ## Safe reclamation Only records with both the durable owner resolution receipt and matching journal diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md new file mode 100644 index 000000000..f851207e1 --- /dev/null +++ b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md @@ -0,0 +1,25 @@ +## ADDED Requirements + +### Requirement: Atomic recovery owner settlement +(P1) The Store SHALL support a checked atomic append for exactly two distinct durable owners, each with an exact expected sequence, so a resolution receipt and its native acknowledgement can commit together. Owner IDs SHALL contain 1–256 bytes; the pair SHALL contain at most 64 events and 16 MiB of encoded events. The decision, event validation, sequences and projectors SHALL share one IMMEDIATE transaction. Rejected decisions or commits SHALL roll back both owners and decision writes; notifications SHALL follow only observed commit success. An unknown completed COMMIT SHALL fence writes and require durable reconciliation rather than repeat filesystem effects. + +#### Scenario: Late second-owner conflict +- **WHEN** the second owner has changed after the first owner's events and projections have been applied inside the transaction +- **THEN** the entire transaction rolls back, including decision writes and both owner sequences, and neither owner emits a commit notification + +#### Scenario: Invalid second-owner event or projection +- **WHEN** validation or a projector rejects the second owner's settlement +- **THEN** no event or projection from either owner commits and the writer remains available for a corrected request + +#### Scenario: Rejected database commit +- **WHEN** a deferred constraint rejects COMMIT after both owner batches have been applied +- **THEN** both batches and decision writes roll back, no commit notification is emitted and a corrected request can succeed + +#### Scenario: Committed settlement loses its reply +- **WHEN** SQLite persists both owners but reports a COMMIT error +- **THEN** the Store emits no success notification and fences new writes +- **AND** reopening preserves both original records for reconciliation without duplicating either owner + +#### Scenario: Bounded decision refuses +- **WHEN** a checked pair has duplicate, empty or oversized owner IDs, an invalid expected sequence, too many events or an oversized encoding +- **THEN** it returns an explicit nonfatal refusal and commits no decision or event changes diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 5cfc3d508..1cfee72f6 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -16,13 +16,43 @@ Task 2.4 evidence now includes real service kill/reload/restage/clear, disabled ## 3. Non-git snapshots -- [ ] 3.0 Review the concrete [managed-recovery ownership protocol](managed-recovery.md), including exact native acknowledgement, legacy journal limits and two-phase reclamation, before changing recovery admission. +- [x] 3.0 Review the concrete [managed-recovery ownership protocol](managed-recovery.md), including exact native acknowledgement, legacy journal limits and two-phase reclamation, before changing recovery admission. Approved on 2026-11-06: "Approve this protocol and implement." - [x] 3.1 Add private content-addressed BLAKE3 objects and Location-bound manifests behind the snapshot interface, including ignore/size/metadata reporting and same-size/same-mtime verification. - [ ] 3.2 Implement conflict-aware owned restore and recoverable backups/journal, preserving Git behavior, protected paths, symlink/hardlink safety and interruption evidence. - [ ] 3.3 Integrate 200 MiB cap, visible unavailable/overflow state, reference-aware retention and staged/branch/recovery protections. - [ ] 3.4 Test real non-git editing/restoration, limits, deletes/renames, unsupported files, concurrent Session/editor changes, crashes and native filesystem privacy. +### Managed recovery implementation and acceptance map + +These steps expand 3.2–3.4 and the remaining native cuts in 2.4. They do not +select the protocol by themselves. The user approved 3.0 on 2026-11-06. The +source map was inspected on main `db9e171e`; this breakdown is not evidence of +recovery implementation. + +- [x] M1 Record the user's 3.0 decision in managed-recovery.md and reconcile any requested revisions with the snapshot spec delta before implementation. Approved without protocol revisions on 2026-11-06. +- [ ] M2 Add immutable snapshot-worker origin and request ownership to the runtime activity and snapshot ports. Retain actual lifecycle/checkout/store/mutation owners inside the synchronous worker through its last filesystem call. Test request disposal, cancellation, shutdown and a foreign/live worker; never reuse binary-document or generic-process acknowledgement authority. +- [ ] M3 Add versioned origin-bound Git and file-copy intents, preserved displaced preimages and resolution-intent records. Keep legacy journals inspectable without inventing origin proof. Test missing/corrupt records, private namespace replacement, aliases, metadata and Location mismatch before admitting effects. +- [ ] M4 Implement bounded keep/rollback planning and fingerprint revalidation under exact native locks. Preserve unrelated edits, refuse the complete plan on conflicts, and retain any interrupted resolution as independent evidence. Test keep and rollback after actual partial namespace replacement, including staged undo and inactive branches. +- [ ] M5 Atomically validate and commit the exact origin acknowledgement and resolution receipt at the Store writer. Determine the bounded owner aggregates before extending the single-aggregate append API. Test late CAS/projector/validation failure rollback, absence of uncommitted notifications and committed-but-lost writer replies; repeat request lookup must never mutate files again. + +M5 prerequisite: `Store::append_pair_checked` implements the bounded two-owner +transaction with exact sequences, post-commit notifications and rollback/lost-ack +regressions. Native origin validation and resolution request ownership are not +implemented by that primitive; M5 remains open until the runtime uses it with +exact native proof and actual restart/retry acceptance. + +- [ ] M6 Publish journal acknowledgement only after observed durable Store settlement. Add reference/reader/lease-aware eligibility and durable two-phase reclamation with retained retry receipts. Test cap pressure, disabled/zero retention, incomplete reference views and crashes at writer/journal/reclaim boundaries without deleting unresolved evidence. +- [ ] M7 Add owner recovery review/confirm/lookup HTTP routes, typed CLI and SDK operations, and TUI recovery entry from rewind unavailability. Regenerate OpenAPI/SDK/command artifacts; exercise the real clients with no inference or transcript movement and with stale/foreign request refusal. +- [ ] M8 Run process-kill cuts before/after intent, stage preparation, every exchange, progress, filesystem completion, writer commit, journal acknowledgement and reclaim. Obtain Linux x64/ARM and macOS Intel/ARM positives/denials, explicit Windows results/limits and shared Git/binary regressions; update evidence and perform substantive code/security review before accepting parent tasks. + +Implementation source map: + +- `cyber-server/src/runtime/activity.rs` owns native activity records; `binary_documents.rs` retains a real synchronous worker's resources and `binary_recovery.rs` demonstrates document-specific recovery proof. Snapshot workers require their own origin/proof rather than calling binary recovery. +- `cyber-server/src/runtime/host.rs::Snapshots` currently exposes restore and read-only recovery inspection; `rewind_preview.rs` owns reviewed confirmation. A dedicated owner recovery admission must work while ordinary admission remains fenced. +- `cyber-snapshot/src/copy/restore.rs` has version-1 intents without native activity origin; `copy/storage.rs` owns private descriptor-relative I/O. `restore.rs` and `git_history.rs` cover Git restore and historical exclusions. Keep historical evidence readable as the new protocol is added. +- `cyber-store/src/store.rs::append_checked` currently appends one aggregate. Separate independent appends cannot establish atomic acknowledgement plus resolution ownership. + ## 4. Rules and context inspection - [ ] 4.1 Implement trusted path-scoped rule discovery/precedence, validated globs, overrides and bounded instruction assembly without changing `instructions` array compatibility. From 0d21b0b598c2010c5d652e857a26cee9c9c2ac5f Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 12:48:29 -0400 Subject: [PATCH 02/87] Retain owned non-Git restore workers through acknowledgement --- .config/nextest.toml | 2 +- crates/cyber-cli/tests/branches_cli.rs | 94 ++++ crates/cyber-core/src/binary/native.rs | 20 + crates/cyber-server/src/runtime/activity.rs | 112 ++++- .../src/runtime/admission_authority.rs | 12 + .../cyber-server/src/runtime/continuation.rs | 2 + crates/cyber-server/src/runtime/host.rs | 15 + crates/cyber-server/src/runtime/mod.rs | 71 ++- crates/cyber-server/src/runtime/rewind.rs | 9 +- .../src/runtime/rewind_preview.rs | 346 +++++++++------ .../cyber-server/src/runtime/snapshot_work.rs | 302 +++++++++++++ .../src/runtime/snapshot_work_tests.rs | 412 ++++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 30 ++ crates/cyber-snapshot/src/copy/restore.rs | 365 ++++++++++++++-- crates/cyber-snapshot/src/copy/storage.rs | 10 +- crates/cyber-snapshot/src/lib.rs | 66 +++ crates/cyber-snapshot/tests/file_copy.rs | 46 ++ ...261106000000_snapshot_worker_ownership.sql | 4 + crates/cyber-store/src/migrate.rs | 56 +++ docs/rewind.md | 18 +- .../implementation.md | 65 +++ .../managed-recovery.md | 4 + .../specs/snapshots-checkpoints/spec.md | 29 ++ .../specs/storage-events/spec.md | 5 + .../complete-daily-coding-surfaces/tasks.md | 19 +- 25 files changed, 1893 insertions(+), 221 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_work.rs create mode 100644 crates/cyber-server/src/runtime/snapshot_work_tests.rs create mode 100644 crates/cyber-store/migrations/20261106000000_snapshot_worker_ownership.sql diff --git a/.config/nextest.toml b/.config/nextest.toml index 29cac8003..5c8407bfd 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -162,7 +162,7 @@ binary_id(cyber-account) | binary_id(cyber-snapshot) | binary_id(cyber-snapshot::file_copy_platform) | binary_id(cyber-snapshot::snapshots) -| binary_id(cyber-store) & (test(writer::tests::) | test(store::paired_tests::) | test(migrate::tests::messaging_) | test(migrate::tests::usage_origin_)) +| binary_id(cyber-store) & (test(writer::tests::) | test(store::paired_tests::) | test(migrate::tests::snapshot_worker_) | test(migrate::tests::messaging_) | test(migrate::tests::usage_origin_)) | binary_id(cyber-store::store) | binary_id(cyber-tools) & (test(tools::bash::shell_tests) | test(plugin_host::) | test(browser::) | test(dev_server::) | test(permissions::engine::tests::message_send) | test(permissions::engine::tests::remote_attach) | test(remote_skills) | test(lsp::) | test(mcp::stdio::tests) | test(mcp::roots::tests) | test(mcp::elicitation::tests) | test(mcp::sampling::tests) | test(tools::process::tests) | test(tools::tool_search::tests) | test(mcp::pool::tests) | test(mcp::launch::tests) | test(mcp::http::tests) | test(mcp::oauth) | test(hook_commands::launch::tests) | test(sandboxing::tests) | test(permissions::filesystem) | test(tools::background::tests) | test(tools::monitor::) | test(tools::deliver::tests) | test(tools::powershell::tests) | test(permissions::dry_run) | test(sandboxing::tests::refusal_lines) | test(permissions::engine::tests::sandbox_boundary_needs_an_explicit_user_allow) | test(permissions::engine::tests::workflow_run) | test(permissions::taint::) | test(untrusted::)) | binary_id(cyber-tools::agent_context) diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 1587e28ca..5825b6120 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -732,6 +732,40 @@ fn interrupt_and_wait_idle(env: &Env, id: &str) { } } +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn reviewed_non_git_stage_and_clear_keep_exact_acknowledged_worker_receipts() { + let model = scripted_model::ScriptedModel::start(reply); + let env = Env::new(&model); + std::fs::remove_dir_all(env.root.join("repo/.git")).unwrap(); + let id = env.exec(None, "WRITE-A"); + let first = env.user_ids(&id)[0].clone(); + std::fs::write(env.root.join("repo/owner.txt"), "owner before stage").unwrap(); + let requests = model.requests().len(); + reviewed_apply( + &env, + &id, + &reviewed_selection(&env, &first, "code", "stage"), + ); + assert_eq!(env.file("a.txt"), None); + std::fs::write(env.root.join("repo/owner.txt"), "later owner bytes").unwrap(); + reviewed_apply( + &env, + &id, + &reviewed_selection(&env, &first, "code", "clear"), + ); + assert_eq!(env.file("a.txt").as_deref(), Some("alpha\n")); + assert_eq!(env.file("owner.txt").as_deref(), Some("later owner bytes")); + assert!(session_revert(&env, &id).is_null()); + let catalog = json(&env.cyber(&["api", "GET", &format!("/sessions/{id}/rewind/recovery")])); + let journals = catalog["data"].as_array().unwrap(); + assert_eq!(journals.len(), 2); + for journal in journals { + assert_owned_snapshot_receipt(&env, &id, journal["id"].as_str().unwrap()); + } + assert_eq!(model.requests().len(), requests); +} + #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn reviewed_non_git_code_rewind_uses_private_snapshots_and_keeps_owner_files() { @@ -757,6 +791,7 @@ fn reviewed_non_git_code_rewind_uses_private_snapshots_and_keeps_owner_files() { .clone(); assert_eq!(journals.len(), 1); assert_eq!(journals[0]["filesystem_complete"], true); + assert_owned_snapshot_receipt(&env, &id, journals[0]["id"].as_str().unwrap()); assert!( journals[0]["paths"] .as_array() @@ -772,6 +807,65 @@ fn reviewed_non_git_code_rewind_uses_private_snapshots_and_keeps_owner_files() { assert!(all.to_string().contains(&first)); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { + let namespaces: Vec<_> = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect(); + assert_eq!(namespaces.len(), 1); + let intent: Value = serde_json::from_slice( + &std::fs::read( + namespaces[0] + .join("journals") + .join(journal) + .join("intent.json"), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(intent["version"], 2); + assert_eq!(intent["completed"], true); + assert_eq!(intent["acknowledged"], true); + let origin = &intent["origin"]; + assert_eq!(origin["session_id"], session); + assert_eq!(origin["binding"]["journal_id"], journal); + let databases: Vec<_> = std::fs::read_dir(env.root.join("cyber-home/data")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "db")) + .collect(); + assert_eq!( + databases.len(), + 1, + "fixture must identify the actual channel database" + ); + let db = rusqlite::Connection::open_with_flags( + &databases[0], + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .unwrap(); + let data: String = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [origin["activity_id"].as_str().unwrap()], + |row| row.get(0), + ) + .unwrap(); + let activity: Value = serde_json::from_str(&data).unwrap(); + assert_eq!(activity["snapshot_origin"], *origin); + assert_eq!(activity["status"], "settled"); + assert_eq!(activity["phase"], "snapshot_restore_committed"); + let created: String = db + .query_row( + "SELECT id FROM event WHERE aggregate_id=?1 ORDER BY seq LIMIT 1", + [session], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(origin["session_created_id"], created); +} + #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn reviewed_non_git_cap_warns_once_without_failing_the_conversation() { diff --git a/crates/cyber-core/src/binary/native.rs b/crates/cyber-core/src/binary/native.rs index 99512476c..97fc1afdd 100644 --- a/crates/cyber-core/src/binary/native.rs +++ b/crates/cyber-core/src/binary/native.rs @@ -226,6 +226,15 @@ fn open(root: &Path, relative: &Path, write: bool) -> Result<(Dir, File, Vec Result<(), Error> { + let file = root.try_clone().map_err(|_| Error::Io)?.into_std_file(); + if self.parents.first() != Some(&id(&file)?) { + return Err(Error::Changed); + } + Ok(()) + } + pub fn capture(root: &Path, relative: &Path) -> Result { let (_, file, parents) = open(root, relative, true)?; let m = file.metadata().map_err(|_| Error::Io)?; @@ -453,6 +462,17 @@ mod tests { let target = Target::capture(root.path(), Path::new("target")).unwrap(); (root, target, private) } + #[test] + fn captured_target_cannot_adopt_a_different_pinned_root() { + let (root, target, _) = setup(); + let original = Dir::open_ambient_dir(root.path(), cap_std::ambient_authority()).unwrap(); + assert!(target.validate_root(&original).is_ok()); + let foreign = tempfile::tempdir().unwrap(); + std::fs::write(foreign.path().join("target"), b"before").unwrap(); + let replacement = Target::capture(foreign.path(), Path::new("target")).unwrap(); + assert!(replacement.validate_root(&original).is_err()); + } + #[test] fn exchange_installs_exact_new_inode_preserves_displaced_bytes_and_metadata() { let (root, target, private) = setup(); diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 1a6c297f1..d52feb123 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -11,6 +11,7 @@ tokio::task_local! { static CANCEL: CancellationToken; } pub(super) struct Control { pub source: String, pub binary_document: Option, + pub snapshot_worker: bool, pub admission_sessions: std::collections::HashSet, pub stop: CancellationToken, pub done: CancellationToken, @@ -23,6 +24,10 @@ pub(super) struct Record { /// Trusted synchronous filesystem-only workers; no subprocess can escape this scope. #[serde(default, skip_serializing_if = "Option::is_none")] pub binary_document: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub snapshot_worker: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_origin: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_bindings: Option>, } @@ -59,11 +64,25 @@ impl Scope { inner: &Inner, handle: &Handle, binary_document: Option, + ) -> Result { + let source = handle.state.lock().await.info.id.clone(); + Self::reserve_owner(inner, source, binary_document, false).await + } + pub(super) async fn reserve_snapshot( + inner: &Inner, + source: String, + ) -> Result { + Self::reserve_owner(inner, source, None, true).await + } + async fn reserve_owner( + inner: &Inner, + source: String, + binary_document: Option, + snapshot_worker: bool, ) -> Result { let runtime = Runtime { inner: inner.me.upgrade().ok_or(RuntimeError::ShuttingDown)?, }; - let source = handle.state.lock().await.info.id.clone(); let authority = runtime.capture_child_admission(&source)?; let owned_handle = runtime.inner.handle(&source).await?; let id = cyber_core::ids::new_id("op"); @@ -72,6 +91,8 @@ impl Scope { status: "pending".into(), phase: "reserved".into(), binary_document, + snapshot_worker, + snapshot_origin: None, admission_bindings: Some(authority.bindings.clone()), }; inner @@ -80,6 +101,7 @@ impl Scope { let control = Arc::new(Control { source: source.clone(), binary_document: record.binary_document.clone(), + snapshot_worker, admission_sessions: super::admission_authority::binding_sessions(&authority.bindings), stop: inner.closed.child_token(), done: CancellationToken::new(), @@ -117,6 +139,75 @@ impl Scope { self.launched = true; Ok(()) } + pub(super) fn launch_snapshot( + &mut self, + origin: super::SnapshotOrigin, + ) -> Result<(), RuntimeError> { + if !self.record.snapshot_worker + || self.record.snapshot_origin.is_some() + || self.launched + || origin.version != 1 + || origin.binding.directory.to_str().is_none() + || origin.activity_id != self.id + || origin.session_id != self.control.source + { + return Err(RuntimeError::Conflict( + "Snapshot origin cannot replace another activity".into(), + )); + } + self.record.snapshot_origin = Some(origin); + self.launch() + } + pub(super) fn snapshot_origin(&self) -> Option<&super::SnapshotOrigin> { + self.launched + .then_some(self.record.snapshot_origin.as_ref()) + .flatten() + } + pub(super) fn commit_snapshot( + &mut self, + state: &mut super::SessionState, + sequence: i64, + events: Vec, + ) -> Result<(), RuntimeError> { + let runtime = Runtime { + inner: self.inner.upgrade().ok_or(RuntimeError::ShuttingDown)?, + }; + self.authority.verify(&runtime, &self.control.source)?; + if state.last_seq != sequence + || !self.record.snapshot_worker + || state.info.id != self.control.source + || self.control.stop.is_cancelled() + { + return Err(RuntimeError::Conflict( + "Snapshot commit authority changed".into(), + )); + } + let mut settled = self.record.clone(); + settled.status = "settled".into(); + settled.phase = "snapshot_restore_committed".into(); + settled.admission_bindings = None; + let bindings = self.authority.bindings.clone(); + let acknowledgement = change(&settled); + let owners = [ + (state.info.id.clone(), state.last_seq), + (self.id.clone(), if self.launched { 1 } else { 0 }), + ]; + let (stored, ()) = self.store.append_pair_checked(owners, move |tx| { + if !super::admission_authority::bindings_current(tx, &bindings)? { + return Err(cyber_store::StoreError::InvalidAppend( + super::admission_authority::STALE.into(), + )); + } + Ok(([events, vec![acknowledgement]], ())) + })?; + self.record = settled; + self.settled = true; + for event in &stored[0] { + state.apply(event).map_err(RuntimeError::Corrupt)?; + } + runtime.inner.publish(&stored[0]); + Ok(()) + } pub fn settle(&mut self) -> Result<(), RuntimeError> { self.record.status = "settled".into(); self.record.admission_bindings = None; @@ -210,7 +301,8 @@ impl Inner { } let filesystem_only = controls .iter() - .all(|control| control.binary_document.is_some()); + .all(|control| control.binary_document.is_some() || control.snapshot_worker); + let snapshot_worker = controls.iter().any(|control| control.snapshot_worker); let acknowledged = tokio::time::timeout(std::time::Duration::from_secs(2), async { for control in controls { control.done.cancelled().await; @@ -222,12 +314,16 @@ impl Inner { if !filesystem_only { handle.location_uncertain.store(true, Ordering::SeqCst); } - return Err(RuntimeError::Invalid(if filesystem_only { - "Binary filesystem work is still settling; query its operation before retrying" - .into() - } else { - "Idle operation did not acknowledge interruption; recovery is required".into() - })); + return Err(RuntimeError::Invalid( + if filesystem_only && snapshot_worker { + "Snapshot filesystem work is still settling; inspect recovery evidence before retrying".into() + } else if filesystem_only { + "Binary filesystem work is still settling; query its operation before retrying" + .into() + } else { + "Idle operation did not acknowledge interruption; recovery is required".into() + }, + )); } if handle.location_uncertain.load(Ordering::SeqCst) { return Err(RuntimeError::Invalid( diff --git a/crates/cyber-server/src/runtime/admission_authority.rs b/crates/cyber-server/src/runtime/admission_authority.rs index a4c4c2787..e7ee91322 100644 --- a/crates/cyber-server/src/runtime/admission_authority.rs +++ b/crates/cyber-server/src/runtime/admission_authority.rs @@ -196,6 +196,18 @@ impl AdmissionAuthority { } impl Runtime { + pub(super) fn admitted_bindings( + &self, + source: &str, + attempt: Option<&super::names::Resumed>, + ) -> Result>, RuntimeError> { + let bindings = self.callback_admission_bindings(source)?; + if let Some(captured) = attempt.and_then(|attempt| attempt.admission_bindings.as_ref()) { + self.verify_admission_bindings(captured)?; + } + Ok(bindings) + } + pub(super) fn verify_admission_bindings( &self, bindings: &[Binding], diff --git a/crates/cyber-server/src/runtime/continuation.rs b/crates/cyber-server/src/runtime/continuation.rs index f56a64c19..3312cb986 100644 --- a/crates/cyber-server/src/runtime/continuation.rs +++ b/crates/cyber-server/src/runtime/continuation.rs @@ -42,6 +42,7 @@ impl Runtime { if let Some(receipt) = existing_receipt(&state, &message, &digest)? { return Ok(receipt); } + self.inner.ensure_snapshot_admission(&handle, id)?; if state.info.parent_id.is_none() { drop(lifecycle); return self.admit_prompt(id, admission, files).await; @@ -242,6 +243,7 @@ impl Runtime { ) -> Result<(), RuntimeError> { state.ensure_worktree_ready()?; if state.child_continuation_unknown + || handle.snapshot_active.load(Ordering::SeqCst) || handle.location_uncertain.load(Ordering::SeqCst) || state.calls.values().any(|call| { matches!( diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 3df583bce..7a0cd6bc2 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -1429,6 +1429,21 @@ pub struct SnapshotRecovery { /// Working-tree snapshots for a Location. pub trait Snapshots: Send + Sync { + /// Only synchronous filesystem hosts retaining runtime-issued worker ownership. + fn owned_restore_available(&self, _directory: &str) -> bool { + false + } + fn restore_owned( + &self, + _request: super::SnapshotRestoreRequest, + _worker: super::SnapshotWorker, + ) -> BoxFuture<'_, Result> { + Box::pin(async { + Err(RestoreError::Failed( + "Owned snapshot restore is unsupported".into(), + )) + }) + } /// Actionable unavailability retained after a one-time automatic warning. fn unavailable_reason(&self, _directory: &str) -> Option { None diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 365e710db..7829c9fc6 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -122,10 +122,15 @@ mod model; mod requests; mod rewind; mod rewind_preview; +mod snapshot_work; pub use rewind_preview::{ RewindAction, RewindCodePreview, RewindConfirmation, RewindEffect, RewindPreview, RewindSelection, }; +pub use snapshot_work::{ + SnapshotBinding, SnapshotCompletion, SnapshotNativeGuard, SnapshotNativeResult, SnapshotOrigin, + SnapshotRestoreRequest, SnapshotWork, SnapshotWorker, +}; mod routing; mod sandbox_events; pub use sandbox_events::SandboxEvent; @@ -728,6 +733,7 @@ pub struct SessionPage { pub(crate) struct Handle { pub state: Mutex, + pub snapshot_active: std::sync::atomic::AtomicBool, pub pending_compaction: StdMutex>>, pub title_requested: std::sync::atomic::AtomicBool, /// Set by a reject without feedback or a dismissed question: stop after the current tool group. @@ -1613,6 +1619,30 @@ impl Runtime { .await } + async fn admission_effects( + &self, + session_id: &str, + admission: &Admission, + check: &PromptCheck, + message_id: &str, + digest: &str, + ) -> Result<(PromptEffects, PromptAttachments), RuntimeError> { + match check { + PromptCheck::Run | PromptCheck::RunWithFiles(_) => { + self.prompt_hooks(session_id, admission, check.files(), message_id, digest) + .await + } + PromptCheck::Checked(context) => Ok(( + PromptEffects { + context: context.clone(), + stop: None, + }, + self.checked_attachments(session_id, &admission.parts) + .await?, + )), + } + } + /// Admit with the input's `untrusted` record. A Channel event is always scanned /// here, so no Channel admission path can skip the detector. async fn admit_recorded( @@ -1630,31 +1660,22 @@ impl Runtime { .clone() .unwrap_or_else(|| cyber_core::ids::new_id("msg")); let digest = admission.digest(check.files()); + { + let handle = self.inner.handle(session_id).await?; + let state = handle.state.lock().await; + if let Some(receipt) = existing_receipt(&state, &message_id, &digest)? { + return Ok(receipt); + } + self.inner.ensure_snapshot_admission(&handle, session_id)?; + } // A denied prompt leaves no durable trace, including a staged revert. let prompt = check.runs_hooks(); - let (effects, attachments) = match &check { - PromptCheck::Run | PromptCheck::RunWithFiles(_) => { - self.prompt_hooks(session_id, &admission, check.files(), &message_id, &digest) - .await? - } - PromptCheck::Checked(context) => ( - PromptEffects { - context: context.clone(), - stop: None, - }, - self.checked_attachments(session_id, &admission.parts) - .await?, - ), - }; + let (effects, attachments) = self + .admission_effects(session_id, &admission, &check, &message_id, &digest) + .await?; let _admission = self.inner.open().await?; self.inner.ensure_admission_open(session_id)?; - let admission_bindings = self.callback_admission_bindings(session_id)?; - if let Some(bindings) = attempt - .as_ref() - .and_then(|attempt| attempt.admission_bindings.as_ref()) - { - self.verify_admission_bindings(bindings)?; - } + let admission_bindings = self.admitted_bindings(session_id, attempt.as_ref())?; let handle = self.inner.handle(session_id).await?; handle.state.lock().await.ensure_worktree_ready()?; self.inner.commit_staged_revert(&handle).await?; @@ -1674,6 +1695,7 @@ impl Runtime { blobs.keep(); return Ok(receipt); } + self.inner.ensure_snapshot_admission(&handle, session_id)?; // The state lock makes accepted input visible before settlement chooses cleanup. if self.child_is_settling(session_id) { return Err(RuntimeError::Busy(session_id.into())); @@ -2558,12 +2580,17 @@ impl Inner { RuntimeError::Invalid("browser-managed Session capacity reached".into()) })?; } + let snapshot_source = id.to_owned(); + let snapshot_unknown = self.store.read(move |db| { + Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM event e WHERE e.type='native.activity.changed.1' AND json_extract(e.data,'$.session_id')=?1 AND json_extract(e.data,'$.snapshot_worker')=1 AND json_extract(e.data,'$.status')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq))", [snapshot_source], |row| row.get::<_, bool>(0))?) + })?; let handle = Arc::new(Handle { title_requested: std::sync::atomic::AtomicBool::new(!state.info.default_title), state: Mutex::new(state), + snapshot_active: std::sync::atomic::AtomicBool::new(false), pending_compaction: StdMutex::default(), halt: std::sync::atomic::AtomicBool::new(false), - location_uncertain: std::sync::atomic::AtomicBool::new(false), + location_uncertain: std::sync::atomic::AtomicBool::new(snapshot_unknown), recent_calls: StdMutex::default(), auto_blocks: Mutex::new(0), stop_continuations: std::sync::atomic::AtomicUsize::new(0), diff --git a/crates/cyber-server/src/runtime/rewind.rs b/crates/cyber-server/src/runtime/rewind.rs index c51c299ad..fb25f9e28 100644 --- a/crates/cyber-server/src/runtime/rewind.rs +++ b/crates/cyber-server/src/runtime/rewind.rs @@ -107,6 +107,7 @@ impl Inner { /// is of the current path. pub(crate) async fn commit_staged_revert(&self, handle: &Handle) -> Result<(), RuntimeError> { let mut state = handle.state.lock().await; + self.ensure_snapshot_admission(handle, &state.info.id)?; let Some(revert) = state.revert.clone() else { return Ok(()); }; @@ -123,7 +124,13 @@ impl Inner { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .contains_key(session_id); - if running { + let snapshot = self + .activities + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .any(|control| control.source == session_id && control.snapshot_worker); + if running || snapshot { Err(RuntimeError::Busy(session_id.into())) } else { Ok(()) diff --git a/crates/cyber-server/src/runtime/rewind_preview.rs b/crates/cyber-server/src/runtime/rewind_preview.rs index 6fabfb3f0..11df2eb0b 100644 --- a/crates/cyber-server/src/runtime/rewind_preview.rs +++ b/crates/cyber-server/src/runtime/rewind_preview.rs @@ -364,142 +364,216 @@ impl Runtime { } self.inner.ensure_idle(session_id)?; let handle = self.inner.handle(session_id).await?; - self.inner - .with_idle_location(&handle, async { - self.ensure_no_background(session_id)?; - let before = handle.state.lock().await.clone(); - let preview = self - .inner - .build_rewind_preview(&before, &confirmation.selection) - .await?; - if preview.confirmation != confirmation.confirmation { - return Err(stale()); - } - let authority = self.capture_child_admission(session_id)?; - let runtime = self.clone(); - let source = session_id.to_owned(); - let authorize = std::sync::Arc::new(move || { - authority - .verify(&runtime, &source) - .map_err(|error| error.to_string()) - }); - let mut state = handle.state.lock().await; - self.inner.ensure_idle(session_id)?; - self.ensure_no_background(session_id)?; - validate(&state, &confirmation.selection)?; - if state.last_seq != before.last_seq || handle.is_compacting() { - return Err(stale()); - } - let paths = staged_paths(&preview)?; - let staged_diff = if preview.action == RewindAction::Stage { - if let Some(code) = &preview.code { - let baseline = state - .revert - .as_ref() - .and_then(|r| r.baseline.as_deref()) - .unwrap_or(&code.baseline); - Some( - self.inner - .preview_diff(&state.info.directory, baseline, &code.target) - .await, - ) - } else { - None - } - } else { - None - }; - let (baseline, applied, diff) = if let Some(code) = &preview.code { - let restored = if preview.action == RewindAction::Clear { - let paths = state - .revert - .as_ref() - .and_then(|r| r.paths.clone()) - .ok_or_else(stale)?; - self.inner - .options - .snapshots - .restore_reviewed_paths( - &state.info.directory, - &code.target, - &code.recorded, - &code.baseline, - authorize, - paths, - ) - .await - } else { - self.inner - .options - .snapshots - .restore_reviewed( - &state.info.directory, - &code.target, - &code.recorded, - &code.baseline, - authorize, - ) - .await - }; - let applied = match restored { - Ok(_) => code.target.clone(), - Err(super::RestoreError::Stale) => return Err(stale()), - Err(super::RestoreError::Conflict(paths)) => { - return Err(RuntimeError::RewindConflict(paths)); - } - Err(super::RestoreError::Failed(message)) => { - handle - .location_uncertain - .store(true, std::sync::atomic::Ordering::SeqCst); - return Err(RuntimeError::Invalid(message)); - } - }; - ( - state - .revert - .as_ref() - .and_then(|r| r.baseline.clone()) - .or_else(|| Some(code.baseline.clone())), - Some(applied), - staged_diff.unwrap_or_else(|| { - code.files.iter().map(|file| file.patch.as_str()).collect() - }), - ) - } else { - authorize().map_err(RuntimeError::Invalid)?; - (None, None, String::new()) - }; - let revert = RevertState { + let directory = handle.state.lock().await.info.directory.clone(); + let owned = confirmation.selection.target != RevertTarget::Conversation + && self + .inner + .options + .snapshots + .owned_restore_available(&directory); + if owned { + self.confirm_rewind(&handle, confirmation, true).await + } else { + self.inner + .with_idle_location(&handle, self.confirm_rewind(&handle, confirmation, false)) + .await + } + } + + async fn restore_reviewed_code<'a>( + &self, + handle: &'a std::sync::Arc, + state: tokio::sync::MutexGuard<'a, SessionState>, + preview: &RewindPreview, + authorize: std::sync::Arc Result<(), String> + Send + Sync>, + owned: bool, + ) -> Result< + ( + tokio::sync::MutexGuard<'a, SessionState>, + Option, + ), + RuntimeError, + > { + let code = preview.code.as_ref().ok_or_else(stale)?; + let paths = if preview.action == RewindAction::Clear { + Some( + state + .revert + .as_ref() + .and_then(|r| r.paths.clone()) + .ok_or_else(stale)?, + ) + } else { + None + }; + if owned { + let request = super::SnapshotRestoreRequest { + directory: state.info.directory.clone(), + target: code.target.clone(), + recorded: code.recorded.clone(), + expected: code.baseline.clone(), + paths, + }; + let worker = self.snapshot_worker(handle, &state).await?; + drop(state); + let completed = self + .inner + .options + .snapshots + .restore_owned(request, worker) + .await + .map_err(restore_failure)?; + return Ok((handle.state.lock().await, Some(completed))); + } + let restored = if let Some(paths) = paths { + self.inner + .options + .snapshots + .restore_reviewed_paths( + &state.info.directory, + &code.target, + &code.recorded, + &code.baseline, + authorize, paths, - message_id: preview.message_id, - target: preview.target, - baseline, - applied, - diff, - }; - let events = match preview.action { - RewindAction::Commit => vec![ - revert_event(session_id, &revert, RevertPhase::Stage), - revert_event(session_id, &revert, RevertPhase::Commit), - ], - RewindAction::Stage => { - vec![revert_event(session_id, &revert, RevertPhase::Stage)] - } - RewindAction::Clear => { - vec![revert_event(session_id, &revert, RevertPhase::Clear)] - } - }; - let committed = self.inner.commit_locked(&mut state, events); - if let Err(error) = committed { - if preview.code.is_some() { - handle - .location_uncertain - .store(true, std::sync::atomic::Ordering::SeqCst); - } - return Err(error); - } - Ok(revert) - }) - .await + ) + .await + } else { + self.inner + .options + .snapshots + .restore_reviewed( + &state.info.directory, + &code.target, + &code.recorded, + &code.baseline, + authorize, + ) + .await + }; + if matches!(restored, Err(super::RestoreError::Failed(_))) { + handle + .location_uncertain + .store(true, std::sync::atomic::Ordering::SeqCst); + } + restored.map_err(restore_failure)?; + Ok((state, None)) + } + + async fn confirm_rewind( + &self, + handle: &std::sync::Arc, + confirmation: &RewindConfirmation, + owned: bool, + ) -> Result { + let session_id = handle.state.lock().await.info.id.clone(); + let session_id = session_id.as_str(); + self.ensure_no_background(session_id)?; + let before = handle.state.lock().await.clone(); + let preview = self + .inner + .build_rewind_preview(&before, &confirmation.selection) + .await?; + if preview.confirmation != confirmation.confirmation { + return Err(stale()); + } + let authority = self.capture_child_admission(session_id)?; + let runtime = self.clone(); + let source = session_id.to_owned(); + let authorize = std::sync::Arc::new(move || { + authority + .verify(&runtime, &source) + .map_err(|error| error.to_string()) + }); + let mut state = handle.state.lock().await; + self.inner.ensure_idle(session_id)?; + self.ensure_no_background(session_id)?; + validate(&state, &confirmation.selection)?; + if state.last_seq != before.last_seq || handle.is_compacting() { + return Err(stale()); + } + let paths = staged_paths(&preview)?; + let staged_diff = if preview.action == RewindAction::Stage { + if let Some(code) = &preview.code { + let baseline = state + .revert + .as_ref() + .and_then(|r| r.baseline.as_deref()) + .unwrap_or(&code.baseline); + Some( + self.inner + .preview_diff(&state.info.directory, baseline, &code.target) + .await, + ) + } else { + None + } + } else { + None + }; + let mut completion = None; + let (baseline, applied, diff) = if let Some(code) = &preview.code { + let (restored_state, completed) = self + .restore_reviewed_code(handle, state, &preview, authorize, owned) + .await?; + state = restored_state; + completion = completed; + let applied = code.target.clone(); + ( + state + .revert + .as_ref() + .and_then(|r| r.baseline.clone()) + .or_else(|| Some(code.baseline.clone())), + Some(applied), + staged_diff + .unwrap_or_else(|| code.files.iter().map(|file| file.patch.as_str()).collect()), + ) + } else { + authorize().map_err(RuntimeError::Invalid)?; + (None, None, String::new()) + }; + let revert = RevertState { + paths, + message_id: preview.message_id, + target: preview.target, + baseline, + applied, + diff, + }; + let events = match preview.action { + RewindAction::Commit => vec![ + revert_event(session_id, &revert, RevertPhase::Stage), + revert_event(session_id, &revert, RevertPhase::Commit), + ], + RewindAction::Stage => { + vec![revert_event(session_id, &revert, RevertPhase::Stage)] + } + RewindAction::Clear => { + vec![revert_event(session_id, &revert, RevertPhase::Clear)] + } + }; + let committed = if let Some(completed) = completion { + completed.commit(&mut state, events) + } else { + self.inner.commit_locked(&mut state, events).map(drop) + }; + if let Err(error) = committed { + if preview.code.is_some() { + handle + .location_uncertain + .store(true, std::sync::atomic::Ordering::SeqCst); + } + return Err(error); + } + Ok(revert) + } +} + +fn restore_failure(error: super::RestoreError) -> RuntimeError { + match error { + super::RestoreError::Stale => stale(), + super::RestoreError::Conflict(paths) => RuntimeError::RewindConflict(paths), + super::RestoreError::Failed(message) => RuntimeError::Invalid(message), } } diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs new file mode 100644 index 000000000..385c37455 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -0,0 +1,302 @@ +//! Runtime-issued ownership for synchronous snapshot filesystem workers. +//! The blocking worker retains Session, checkout and lifecycle ownership even +//! after its requester disappears. A journal is not authority to settle a process. +use super::{Handle, Inner, RestoreError, Runtime, RuntimeError, SessionState, activity}; +use cyber_store::NewEvent; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use tokio::sync::OwnedRwLockReadGuard; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SnapshotBinding { + pub directory: PathBuf, + pub root_identity: String, + pub namespace_identity: String, + pub journal_id: String, + pub journal_identity: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SnapshotOrigin { + pub version: u32, + pub session_id: String, + pub session_created_id: String, + pub activity_id: String, + pub worktree_id: Option, + pub binding: SnapshotBinding, +} + +pub struct SnapshotRestoreRequest { + pub directory: String, + pub target: String, + pub recorded: String, + pub expected: String, + pub paths: Option>, +} + +/// Native locks and descriptors held through the consuming runtime commit. +pub trait SnapshotNativeGuard: Send { + fn verify(&self, origin: Option<&SnapshotOrigin>) -> Result<(), String>; + /// Called only after observed durable runtime acknowledgement, under retained locks. + fn acknowledge(&mut self, origin: Option<&SnapshotOrigin>) -> Result<(), String>; +} + +pub struct SnapshotNativeResult { + pub paths: Vec, + guard: Box, +} + +impl SnapshotNativeResult { + pub fn new(paths: Vec, guard: Box) -> Self { + Self { paths, guard } + } +} + +pub struct SnapshotWorker { + // Scope disposal records unknown evidence before releasing Session ownership. + scope: activity::Scope, + admission: SnapshotAdmission, + sequence: i64, + runtime: Runtime, + directory: PathBuf, + created_id: String, + worktree_id: Option, + _location: Box, + _lifecycle: OwnedRwLockReadGuard<()>, +} + +struct SnapshotAdmission(Arc); + +impl Drop for SnapshotAdmission { + fn drop(&mut self) { + self.0.snapshot_active.store(false, Ordering::SeqCst); + } +} + +/// Borrowed only by the actual synchronous filesystem callback; cannot be cloned. +pub struct SnapshotWork<'a> { + owner: &'a mut SnapshotWorker, +} + +pub struct SnapshotCompletion { + worker: SnapshotWorker, + native: SnapshotNativeResult, +} + +impl SnapshotWork<'_> { + pub fn origin(&self) -> Option<&SnapshotOrigin> { + self.owner.scope.snapshot_origin() + } + /// Metadata for pre-effect storage budgeting only; this does not publish or + /// authorize the origin. Source effects still require a successful `begin`. + pub fn preview_origin(&self, binding: SnapshotBinding) -> SnapshotOrigin { + SnapshotOrigin { + version: 1, + session_id: self.owner.scope.control.source.clone(), + session_created_id: self.owner.created_id.clone(), + activity_id: self.owner.scope.id().to_owned(), + worktree_id: self.owner.worktree_id.clone(), + binding, + } + } + pub fn authorize(&self) -> Result<(), String> { + if self.owner.scope.control.stop.is_cancelled() { + return Err("Snapshot worker cancelled; retain its journal before retrying".into()); + } + self.owner + .scope + .authority() + .verify(&self.owner.runtime, &self.owner.scope.control.source) + .map_err(|error| error.to_string())?; + let source = self.owner.scope.control.source.clone(); + let sequence = self.owner.sequence; + let current = self + .owner + .runtime + .inner + .store + .aggregate_seq(&source) + .map_err(|error| error.to_string())?; + if current != Some(sequence) { + return Err("Snapshot Session changed before its filesystem effect".into()); + } + Ok(()) + } + + /// Keep Session changes outside the exact synchronous namespace mutation. + /// The longer-running worker keeps a separate admission owner so Stop and + /// read-only inspection need not wait on a held Session projection mutex. + pub fn effect(&self, effect: impl FnOnce() -> Result) -> Result { + if self.origin().is_none() { + return Err("Snapshot source effects require a durable worker origin".into()); + } + let state = self.owner.admission.0.state.blocking_lock(); + self.authorize()?; + if state.last_seq != self.owner.sequence { + return Err("Snapshot Session projection changed before its filesystem effect".into()); + } + effect() + } + + /// Persist immutable runtime origin before the first intent or source effect. + pub fn begin(&mut self, binding: SnapshotBinding) -> Result { + self.authorize()?; + if binding.directory.to_str().is_none() { + return Err("Snapshot Location path cannot be represented as UTF-8".into()); + } + if binding.directory != self.owner.directory + || [ + &binding.root_identity, + &binding.namespace_identity, + &binding.journal_id, + &binding.journal_identity, + ] + .iter() + .any(|id| id.is_empty() || id.len() > 256) + || !binding.journal_id.starts_with("restore_") + || !binding + .journal_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_') + { + return Err("Snapshot native binding differs from the admitted Location".into()); + } + let origin = self.preview_origin(binding); + self.owner + .scope + .launch_snapshot(origin.clone()) + .map_err(|error| error.to_string())?; + Ok(origin) + } +} + +impl SnapshotWorker { + /// Trusted adapters must perform only synchronous filesystem work here; + /// spawning a process or detached task cannot use this acknowledgement proof. + pub async fn run( + mut self, + work: impl FnOnce(&mut SnapshotWork<'_>) -> Result + + Send + + 'static, + ) -> Result { + let _cancel_on_disposal = self.scope.control.stop.clone().drop_guard(); + let joined = tokio::task::spawn_blocking(move || { + let native = work(&mut SnapshotWork { owner: &mut self })?; + native + .guard + .verify(self.scope.snapshot_origin()) + .map_err(RestoreError::Failed)?; + Ok(SnapshotCompletion { + worker: self, + native, + }) + }); + let completed = joined.await.map_err(|_| RestoreError::Failed( + "Snapshot worker did not acknowledge completion; inspect retained recovery evidence".into() + ))??; + _cancel_on_disposal.disarm(); + Ok(completed) + } +} + +impl SnapshotCompletion { + pub(crate) fn commit( + mut self, + state: &mut SessionState, + events: Vec, + ) -> Result<(), RuntimeError> { + self.native + .guard + .verify(self.worker.scope.snapshot_origin()) + .map_err(RuntimeError::Conflict)?; + self.worker + .scope + .commit_snapshot(state, self.worker.sequence, events)?; + self.native + .guard + .acknowledge(self.worker.scope.snapshot_origin()) + .map_err(RuntimeError::Conflict) + } +} + +impl Runtime { + pub(crate) async fn snapshot_worker( + &self, + handle: &Arc, + state: &SessionState, + ) -> Result { + let lifecycle = self.memory_mutation_lease().await?; + self.inner.ensure_idle(&state.info.id)?; + self.ensure_no_background(&state.info.id)?; + self.ensure_user_child_outcome(handle, state)?; + self.inner.ensure_admission_open(&state.info.id)?; + handle + .snapshot_active + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .map_err(|_| RuntimeError::Busy(state.info.id.clone()))?; + let admission = SnapshotAdmission(handle.clone()); + let scope = activity::Scope::reserve_snapshot(&self.inner, state.info.id.clone()).await?; + let lease = self + .inner + .claim_location(&state.info, false, scope.control.stop.clone()) + .await?; + if lease.worktree_id != state.info.worktree_id { + return Err(RuntimeError::Conflict( + "Snapshot checkout binding changed".into(), + )); + } + let directory = std::fs::canonicalize(&state.info.directory) + .map_err(|error| RuntimeError::Invalid(error.to_string()))?; + let created_id = self + .inner + .store + .read_events(&state.info.id, -1, 1)? + .events + .first() + .ok_or_else(|| RuntimeError::Conflict("Snapshot Session origin is missing".into()))? + .id + .clone(); + let worktree_id = state.info.worktree_id.clone(); + Ok(SnapshotWorker { + scope, + runtime: self.clone(), + directory, + created_id, + worktree_id, + _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, + _lifecycle: lifecycle, + admission, + sequence: state.last_seq, + }) + } +} + +impl Inner { + pub(super) fn ensure_snapshot_admission( + &self, + handle: &Handle, + session: &str, + ) -> Result<(), RuntimeError> { + if handle.snapshot_active.load(Ordering::SeqCst) { + return Err(RuntimeError::Busy(session.into())); + } + let source = session.to_owned(); + let unresolved = self.store.read(move |db| { + Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM event e WHERE e.type='native.activity.changed.1' AND json_extract(e.data,'$.session_id')=?1 AND json_extract(e.data,'$.snapshot_worker')=1 AND json_extract(e.data,'$.status')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq))", [source], |row| row.get::<_, bool>(0))?) + })?; + if unresolved { + return Err(RuntimeError::Conflict( + "Snapshot worker outcome requires owned recovery before new input".into(), + )); + } + Ok(()) + } +} + +#[cfg(all(test, any(target_os = "macos", target_os = "linux")))] +#[path = "snapshot_work_tests.rs"] +mod tests; diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs new file mode 100644 index 000000000..2f41277c1 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -0,0 +1,412 @@ +//! Instrumented worker ownership; native journal/namespace proof is tested separately. +use super::super::{Admission, Delivery, RuntimeOptions, events}; +use super::*; +use std::sync::atomic::AtomicUsize; +use std::time::Duration; + +struct Guard { + origin: Option, + acknowledgements: Arc, +} +impl SnapshotNativeGuard for Guard { + fn verify(&self, origin: Option<&SnapshotOrigin>) -> Result<(), String> { + if self.origin.as_ref() != origin { + return Err("foreign worker receipt".into()); + } + Ok(()) + } + fn acknowledge(&mut self, origin: Option<&SnapshotOrigin>) -> Result<(), String> { + self.verify(origin)?; + self.acknowledgements.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} +fn binding(directory: PathBuf) -> SnapshotBinding { + SnapshotBinding { + directory, + root_identity: "instrumented_root".into(), + namespace_identity: "instrumented_namespace".into(), + journal_id: "restore_test".into(), + journal_identity: "instrumented_journal".into(), + } +} +async fn worker(runtime: &Runtime, session: &str) -> SnapshotWorker { + let handle = runtime.inner.handle(session).await.unwrap(); + let state = handle.state.lock().await; + runtime.snapshot_worker(&handle, &state).await.unwrap() +} +fn latest(runtime: &Runtime, session: &str) -> serde_json::Value { + let session = session.to_owned(); + runtime.inner.store.read(move |db| { + let data: String = db.query_row("SELECT data FROM event WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 ORDER BY rowid DESC LIMIT 1", [session], |row| row.get(0))?; + Ok(serde_json::from_str(&data).unwrap()) + }).unwrap() +} +fn restart(runtime: &Runtime, root: &std::path::Path) -> Runtime { + Runtime::new(RuntimeOptions { + store: runtime.inner.store.clone(), + resolver: runtime.inner.resolver.clone(), + tools: runtime.inner.tools.clone(), + global_config_dir: root.join("config"), + shell: "test".into(), + claude_compat: false, + compaction: Default::default(), + retry: Default::default(), + max_steps: None, + today: None, + interactive: false, + snapshots: Arc::new(super::super::NoSnapshots), + }) +} + +#[tokio::test] +async fn non_utf8_location_binding_refuses_before_serializing_a_native_origin() { + use std::os::unix::ffi::OsStringExt; + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let mut owner = worker(&runtime, &session).await; + // Exercise the serialization boundary even on filesystems that reject these + // names themselves. This is not a native path-admission acceptance test. + let directory = temp + .path() + .join(std::ffi::OsString::from_vec(b"root_\xff".to_vec())); + owner.directory = directory.clone(); + let native_binding = binding(directory); + let result = owner + .run(move |work| { + assert!(work.begin(native_binding).unwrap_err().contains("UTF-8")); + assert!(work.origin().is_none()); + Err(RestoreError::Stale) + }) + .await; + assert!(matches!(result, Err(RestoreError::Stale))); + assert_eq!(latest(&runtime, &session)["status"], "settled"); + runtime.shutdown().await; +} + +#[tokio::test] +async fn rejected_origin_append_cannot_authorize_source_effects_even_if_the_adapter_continues() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + runtime + .inner + .store + .set_append_fault(Some(Arc::new(|_, event| { + event.kind == activity::CHANGED + && event.data["snapshot_worker"] == true + && event.data["phase"] == "launching" + && event.data["status"] == "pending" + }))); + let result = owner + .run(move |work| { + assert!(work.begin(native_binding).is_err()); + assert!(work.origin().is_none()); + assert!( + work.effect::<()>(|| panic!("unacknowledged origin authorized a source mutation")) + .is_err() + ); + Err(RestoreError::Failed("origin append refused".into())) + }) + .await; + assert!(result.is_err()); + runtime.inner.store.set_append_fault(None); + assert_eq!(latest(&runtime, &session)["status"], "settled"); + runtime.shutdown().await; +} + +#[tokio::test] +async fn snapshot_completion_commits_session_and_exact_origin_before_native_ack() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + let acknowledgements = Arc::new(AtomicUsize::new(0)); + let seen = acknowledgements.clone(); + let completed = owner + .run(move |work| { + let origin = work.begin(native_binding.clone()).unwrap(); + assert!( + work.begin(native_binding).is_err(), + "origin replacement accepted" + ); + work.effect(|| Ok(())).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: seen, + }), + )) + }) + .await + .unwrap(); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut unrelated = activity::Scope::reserve(&runtime.inner, &handle) + .await + .unwrap(); + let unrelated_id = unrelated.id().to_owned(); + let origin = completed.worker.scope.snapshot_origin().unwrap().clone(); + assert!( + unrelated.launch_snapshot(origin).is_err(), + "generic activity adopted a snapshot origin" + ); + unrelated.launch().unwrap(); + drop(unrelated); + let mut state = handle.state.lock().await; + completed + .commit( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "restored".into(), + usage: None, + cost: None, + model: None, + }, + )], + ) + .unwrap(); + assert_eq!(state.info.title, "restored"); + drop(state); + let record = latest(&runtime, &session); + assert_eq!(record["status"], "settled"); + assert_eq!(record["phase"], "snapshot_restore_committed"); + assert_eq!(record["snapshot_origin"]["session_id"], session); + let created = runtime + .inner + .store + .read_events(&session, -1, 1) + .unwrap() + .events[0] + .id + .clone(); + assert_eq!(record["snapshot_origin"]["session_created_id"], created); + assert!(record.get("binary_document").is_none()); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 1); + let unrelated = runtime + .inner + .store + .read_events(&unrelated_id, -1, 10) + .unwrap(); + assert_eq!(unrelated.events.last().unwrap().data["status"], "unknown"); + assert!( + handle.location_uncertain.load(Ordering::SeqCst), + "snapshot receipt cleared another process's unknown outcome" + ); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); + runtime.shutdown().await; +} + +#[tokio::test] +async fn disposed_snapshot_request_retains_real_worker_owners_and_stop_remains_available() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let mut retry = Admission::text("retained held input", Delivery::Hold); + retry.message_id = Some("msg_snapshot_exact_retry".into()); + let admitted = runtime.admit(&session, retry.clone()).await.unwrap(); + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = permits.clone().acquire_owned().await.unwrap(); + let (started, running) = tokio::sync::oneshot::channel(); + let (finish, gate) = std::sync::mpsc::channel(); + let task = tokio::spawn(owner.run(move |work| { + let _permit = permit; + work.begin(native_binding).unwrap(); + started.send(()).unwrap(); + gate.recv().unwrap(); + assert!(work.authorize().is_err()); + Err(RestoreError::Failed( + "interrupted worker retained its intent".into(), + )) + })); + running.await.unwrap(); + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + assert_eq!(permits.available_permits(), 0); + assert!( + tokio::time::timeout(Duration::from_secs(1), runtime.state(&session)) + .await + .unwrap() + .is_ok() + ); + assert!( + runtime + .admit(&session, Admission::text("must not run", Delivery::Queue)) + .await + .is_err() + ); + assert_eq!( + runtime + .admit(&session, retry.clone()) + .await + .unwrap() + .message_id, + admitted.message_id + ); + assert!( + tokio::time::timeout(Duration::from_secs(3), runtime.interrupt(&session)) + .await + .unwrap() + .is_err() + ); + assert_eq!( + permits.available_permits(), + 0, + "interruption released a live worker owner" + ); + let stopped = tokio::time::timeout(Duration::from_secs(4), runtime.stop_subtree(&session)) + .await + .unwrap() + .unwrap(); + assert_eq!(stopped.status, super::super::SubtreeStopStatus::Unknown); + assert!(stopped.persisted); + assert_eq!( + permits.available_permits(), + 0, + "subtree Stop released a live worker owner" + ); + let shutdown = { + let runtime = runtime.clone(); + tokio::spawn(async move { runtime.shutdown().await }) + }; + tokio::task::yield_now().await; + assert!( + !shutdown.is_finished(), + "shutdown outlived its worker ownership" + ); + finish.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), shutdown) + .await + .unwrap() + .unwrap(); + assert_eq!(permits.available_permits(), 1); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + let restarted = restart(&runtime, temp.path()); + assert!( + restarted + .admit(&session, Admission::text("still fenced", Delivery::Queue)) + .await + .is_err() + ); + restarted.shutdown().await; +} + +#[tokio::test] +async fn snapshot_pre_effect_refusal_does_not_manufacture_an_unknown_native_effect() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let wrong = binding(temp.path().join("foreign")); + let result = owner + .run(move |work| { + assert!( + work.effect::<()>(|| panic!("unbound source effect ran")) + .is_err() + ); + assert!(work.begin(wrong).is_err()); + Err(RestoreError::Stale) + }) + .await; + assert!(matches!(result, Err(RestoreError::Stale))); + let record = latest(&runtime, &session); + assert_eq!(record["status"], "settled"); + assert!(record.get("snapshot_origin").is_none()); + let handle = runtime.inner.handle(&session).await.unwrap(); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); + assert!(!handle.location_uncertain.load(Ordering::SeqCst)); + runtime.shutdown().await; +} + +#[tokio::test] +async fn changed_session_refuses_the_next_source_effect_and_never_acknowledges_it() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + let source = temp.path().join("must_not_change"); + std::fs::write(&source, "owner bytes").unwrap(); + let worker_source = source.clone(); + let (started, running) = tokio::sync::oneshot::channel(); + let (finish, gate) = std::sync::mpsc::channel(); + let task = tokio::spawn(owner.run(move |work| { + work.begin(native_binding).unwrap(); + started.send(()).unwrap(); + gate.recv().unwrap(); + assert!( + work.effect(|| std::fs::write(worker_source, "wrong").map_err(|e| e.to_string())) + .is_err() + ); + Err(RestoreError::Stale) + })); + running.await.unwrap(); + runtime + .rename(&session, "owner changed Session") + .await + .unwrap(); + finish.send(()).unwrap(); + assert!(matches!(task.await.unwrap(), Err(RestoreError::Stale))); + assert_eq!(std::fs::read_to_string(source).unwrap(), "owner bytes"); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + assert_eq!( + runtime.state(&session).await.unwrap().info.title, + "owner changed Session" + ); + runtime.shutdown().await; +} + +#[tokio::test] +async fn rejected_runtime_commit_keeps_origin_unknown_and_never_acknowledges_the_journal() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + let acknowledgements = Arc::new(AtomicUsize::new(0)); + let observed = acknowledgements.clone(); + let completed = owner + .run(move |work| { + let origin = work.begin(native_binding).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: observed, + }), + )) + }) + .await + .unwrap(); + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + let sequence = state.last_seq; + assert!( + completed + .commit( + &mut state, + vec![NewEvent::new("test.unregistered.1", serde_json::json!({}))] + ) + .is_err() + ); + assert_eq!(state.last_seq, sequence); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); + drop(state); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + runtime.shutdown().await; + let restarted = restart(&runtime, temp.path()); + assert!( + restarted + .admit( + &session, + Admission::text("unresolved origin must fence direct input", Delivery::Queue) + ) + .await + .is_err() + ); + assert!( + restarted + .inner + .handle(&session) + .await + .unwrap() + .location_uncertain + .load(Ordering::SeqCst) + ); + restarted.shutdown().await; +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 9f57ec7c3..a7b065bf1 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -64,6 +64,29 @@ pub(crate) struct Store { options: Options, usage: std::cell::Cell>, } + +pub(crate) struct WorkerGuard { + pub store: Store, + pub workspace: tokio::sync::OwnedMutexGuard<()>, + pub origin: Option, +} + +impl cyber_server::runtime::SnapshotNativeGuard for WorkerGuard { + fn verify(&self, origin: Option<&cyber_server::runtime::SnapshotOrigin>) -> Result<()> { + let _retained_workspace = &self.workspace; + if origin != self.origin.as_ref() { + return Err("Snapshot receipt belongs to another worker".into()); + } + self.store.verify_worker_origin(origin) + } + fn acknowledge( + &mut self, + origin: Option<&cyber_server::runtime::SnapshotOrigin>, + ) -> Result<()> { + self.verify(origin)?; + self.store.acknowledge_worker(origin) + } +} impl Store { pub fn open(data: &Path, location: &str, options: Options) -> Result { Self::open_mode(data, location, options, true) @@ -158,6 +181,13 @@ impl Store { }) } fn verify_root(&self) -> Result<()> { + if !std::fs::symlink_metadata(&self.root) + .map_err(fail)? + .file_type() + .is_dir() + { + return Err("Snapshot Location identity changed".into()); + } let fresh = Dir::open_ambient_dir(&self.root, cap_std::ambient_authority()).map_err(fail)?; if storage::dir_identity(&fresh)? != self.root_id { diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 249047743..2923b5b7c 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -2,7 +2,9 @@ use super::storage::{self, Result, fail}; use super::{ENTRY_LIMIT, Entry, RECORD_LIMIT, Store, Tree, now}; use cap_fs_ext::DirExt; -use cyber_server::runtime::{FileDiff, RestoreError}; +use cyber_server::runtime::{ + FileDiff, RestoreError, SnapshotBinding, SnapshotOrigin, SnapshotWork, +}; use serde::{Deserialize, Serialize}; use std::os::unix::fs::PermissionsExt; use std::path::Path; @@ -17,6 +19,8 @@ struct Change { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Journal { + // Version 2 binds the worker, not historical per-file rollback identity. + // Resolution must not substitute Entry hashes for exact native change proof. version: u32, location: std::path::PathBuf, root: String, @@ -24,32 +28,60 @@ struct Journal { completed: bool, changes: Vec, applied: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + origin: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + acknowledged: bool, } impl Store { pub(super) fn ensure_recovered(&self) -> Result<()> { - if self.recovery()?.iter().any(|r| !r.filesystem_complete) { - return Err( - "Snapshot restore requires recovery; private journal and preimages retained".into(), - ); + for id in storage::names(&self.journals, ENTRY_LIMIT)? { + let (_, record) = self.journal(&id)?; + if !record.completed || (record.version == 2 && !record.acknowledged) { + return Err( + "Snapshot restore requires recovery; private journal and preimages retained" + .into(), + ); + } } Ok(()) } + + fn journal(&self, id: &str) -> Result<(cap_std::fs::Dir, Journal)> { + let dir = self.journals.open_dir_nofollow(id).map_err(fail)?; + let record: Journal = + serde_json::from_slice(&storage::read(&dir, "intent.json", RECORD_LIMIT)?) + .map_err(fail)?; + if !matches!( + (record.version, record.origin.is_some()), + (1, false) | (2, true) + ) || record.location != self.root + || record.root != self.root_id + || record.changes.len() > ENTRY_LIMIT + || (record.version == 1 && record.acknowledged) + { + return Err("Invalid snapshot recovery journal".into()); + } + if let Some(origin) = &record.origin { + let binding = &origin.binding; + if origin.version != 1 + || binding.directory != self.root + || binding.root_identity != self.root_id + || binding.journal_id != id + || binding.namespace_identity != storage::dir_identity(&self.journals)? + || binding.journal_identity != storage::dir_identity(&dir)? + { + return Err("Snapshot journal differs from its native origin".into()); + } + } + Ok((dir, record)) + } pub fn recovery(&self) -> Result> { let mut result = vec![]; let mut response_bytes = 32usize; for id in storage::names(&self.journals, ENTRY_LIMIT)? { - let dir = self.journals.open_dir_nofollow(&id).map_err(fail)?; - let record: Journal = - serde_json::from_slice(&storage::read(&dir, "intent.json", RECORD_LIMIT)?) - .map_err(fail)?; - if record.version != 1 - || record.location != self.root - || record.root != self.root_id - || record.changes.len() > ENTRY_LIMIT - { - return Err("Invalid snapshot recovery journal".into()); - } + let (_, record) = self.journal(&id)?; let view = super::Recovery { id, filesystem_complete: record.completed, @@ -206,42 +238,47 @@ impl Store { expected: &str, paths: Option<&[String]>, authorize: &(dyn Fn() -> Result<()> + Send + Sync), + ) -> std::result::Result, RestoreError> { + self.restore_worker(target, recorded, expected, paths, authorize, None) + } + + pub fn restore_worker( + &self, + target: &str, + recorded: &str, + expected: &str, + paths: Option<&[String]>, + authorize: &(dyn Fn() -> Result<()> + Send + Sync), + mut worker: Option<&mut SnapshotWork<'_>>, ) -> std::result::Result, RestoreError> { authorize().map_err(RestoreError::Failed)?; + if let Some(worker) = worker.as_ref() { + worker.authorize().map_err(RestoreError::Failed)?; + } let (_, changes) = self.plan(target, recorded, expected, paths)?; if changes.is_empty() { return Ok(vec![]); } - let record_bytes = serde_json::to_vec(&changes) - .map_err(|e| RestoreError::Failed(e.to_string()))? - .len() as u64; - let reserve = changes - .iter() - .map(|c| { - c.before - .as_ref() - .map_or(0, |e| e.length) - .max(c.after.as_ref().map_or(0, |e| e.length)) - }) - .try_fold( - record_bytes.saturating_mul(2).saturating_add(4096), - |sum, length| sum.checked_add(length).ok_or("Restore size overflow"), - ) - .map_err(|e| RestoreError::Failed(e.into()))?; - self.reserve(reserve).map_err(RestoreError::Failed)?; - // The directory is unique. A crash before publishing its intent is also - // unreadable recovery evidence and fails closed on the next open. let id = format!("restore_{}", ulid::Ulid::new()); + let mut journal = self + .prepare_journal(changes, &id, worker.as_deref()) + .map_err(RestoreError::Failed)?; + // An unpublished/incomplete directory also remains recovery evidence. let dir = storage::private_child(&self.journals, &id).map_err(RestoreError::Failed)?; - let mut journal = Journal { - version: 1, - location: self.root.clone(), - root: self.root_id.clone(), - created: now(), - completed: false, - changes, - applied: vec![], - }; + let origin = worker + .as_mut() + .map(|worker| { + worker.begin(SnapshotBinding { + directory: self.root.clone(), + root_identity: self.root_id.clone(), + namespace_identity: storage::dir_identity(&self.journals)?, + journal_id: id.clone(), + journal_identity: storage::dir_identity(&dir)?, + }) + }) + .transpose() + .map_err(RestoreError::Failed)?; + journal.origin = origin; Self::record(&dir, &journal).map_err(RestoreError::Failed)?; for (i, change) in journal.changes.iter().enumerate() { @@ -258,8 +295,22 @@ impl Store { storage::sync(&dir).map_err(RestoreError::Failed)?; for i in 0..journal.changes.len() { authorize().map_err(RestoreError::Failed)?; - self.install(&dir, i, &journal.changes[i]) - .map_err(RestoreError::Failed)?; + if let Some(worker) = worker.as_ref() { + worker + .effect(|| { + self.verify_worker_namespace( + journal + .origin + .as_ref() + .ok_or("Missing snapshot worker origin")?, + )?; + self.install(&dir, i, &journal.changes[i]) + }) + .map_err(RestoreError::Failed)?; + } else { + self.install(&dir, i, &journal.changes[i]) + .map_err(RestoreError::Failed)?; + } journal.applied.push(journal.changes[i].path.clone()); Self::record(&dir, &journal).map_err(RestoreError::Failed)?; } @@ -268,6 +319,110 @@ impl Store { Self::record(&dir, &journal).map_err(RestoreError::Failed)?; Ok(journal.applied) } + + fn prepare_journal( + &self, + changes: Vec, + id: &str, + worker: Option<&SnapshotWork<'_>>, + ) -> Result { + let origin = worker + .map(|worker| { + worker.authorize()?; + Ok::<_, String>(worker.preview_origin(SnapshotBinding { + directory: self.root.clone(), + root_identity: self.root_id.clone(), + namespace_identity: storage::dir_identity(&self.journals)?, + journal_id: id.to_owned(), + journal_identity: "0".repeat(storage::IDENTITY_MAX_BYTES), + })) + }) + .transpose()?; + let mut journal = Journal { + version: if origin.is_some() { 2 } else { 1 }, + location: self.root.clone(), + root: self.root_id.clone(), + created: now(), + // Budget the largest progress/acknowledgement record, including JSON + // escaping, plus its atomic replacement copy, before creating an intent. + completed: false, + applied: changes.iter().map(|change| change.path.clone()).collect(), + changes, + origin, + acknowledged: worker.is_some(), + }; + let bytes = serde_json::to_vec(&journal).map_err(fail)?.len() as u64; + if bytes > RECORD_LIMIT { + return Err("Restore journal exceeds limit".into()); + } + let reserve = journal + .changes + .iter() + .map(|change| { + change + .before + .as_ref() + .map_or(0, |entry| entry.length) + .max(change.after.as_ref().map_or(0, |entry| entry.length)) + }) + .try_fold( + bytes.checked_mul(2).ok_or("Restore size overflow")?, + |sum, length| sum.checked_add(length).ok_or("Restore size overflow"), + )?; + self.reserve(reserve)?; + journal.applied.clear(); + journal.origin = None; + journal.acknowledged = false; + Ok(journal) + } + + pub(super) fn verify_worker_origin(&self, origin: Option<&SnapshotOrigin>) -> Result<()> { + self.verify_root()?; + let Some(origin) = origin else { return Ok(()) }; + self.verify_worker_namespace(origin)?; + let (_, record) = self.journal(&origin.binding.journal_id)?; + if record.origin.as_ref() != Some(origin) || !record.completed { + return Err("Snapshot worker completion differs from its retained intent".into()); + } + Ok(()) + } + + fn verify_worker_namespace(&self, origin: &SnapshotOrigin) -> Result<()> { + self.verify_root()?; + let binding = &origin.binding; + if binding.directory != self.root + || binding.root_identity != self.root_id + || binding.namespace_identity != storage::dir_identity(&self.journals)? + { + return Err("Snapshot worker namespace changed".into()); + } + let fresh = Self::existing( + &self.data_path, + &self.root.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Snapshot worker namespace is missing")?; + if storage::dir_identity(&fresh.journals)? != binding.namespace_identity { + return Err("Snapshot worker namespace route changed".into()); + } + let dir = self + .journals + .open_dir_nofollow(&binding.journal_id) + .map_err(fail)?; + if storage::dir_identity(&dir)? != binding.journal_identity { + return Err("Snapshot worker journal identity changed".into()); + } + Ok(()) + } + pub(super) fn acknowledge_worker(&self, origin: Option<&SnapshotOrigin>) -> Result<()> { + let Some(origin) = origin else { return Ok(()) }; + let (dir, mut journal) = self.journal(&origin.binding.journal_id)?; + if journal.origin.as_ref() != Some(origin) || !journal.completed { + return Err("Snapshot acknowledgement differs from its completed worker".into()); + } + journal.acknowledged = true; + Self::record(&dir, &journal) + } fn stage_metadata(file: &std::fs::File, entry: &Entry) -> Result<()> { rustix::fs::fchown( file, @@ -287,6 +442,7 @@ impl Store { storage::atomic(dir, "intent.json", &bytes) } fn install(&self, dir: &cap_std::fs::Dir, index: usize, change: &Change) -> Result<()> { + self.verify_root()?; if self.current(&change.path)? != change.before { return Err(format!( "Restore source changed; journal retained: {}", @@ -306,6 +462,7 @@ impl Store { Path::new(&change.path), ) .map_err(fail)?; + target.validate_root(&self.work).map_err(fail)?; if target.length.0 != before.length || target.metadata.mode != before.mode || target.metadata.uid != before.uid @@ -357,3 +514,119 @@ impl Store { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quota_includes_escaped_progress_and_atomic_record_replacement() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work"); + std::fs::create_dir(&root).unwrap(); + let mut store = Store::open( + &temp.path().join("data"), + &root.to_string_lossy(), + super::super::Options { + max_file: 65536, + max_total: 10000, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + let path = vec!["\"".repeat(50); 24].join("/") + "/file"; + let changes = vec![Change { + path, + before: None, + after: Some(Entry { + object: blake3::hash(b"").to_hex().to_string(), + length: 0, + mode: 0o644, + uid: 0, + gid: 0, + }), + }]; + // The former changes-only estimate admitted this escaped-path fixture. + let old_estimate = serde_json::to_vec(&changes).unwrap().len() * 2 + 4096; + assert!(old_estimate < 10000); + assert!( + store + .prepare_journal(changes.clone(), "restore_fixture", None) + .is_err() + ); + assert!( + storage::names(&store.journals, ENTRY_LIMIT) + .unwrap() + .is_empty() + ); + store.options.max_total = 20000; + let journal = store + .prepare_journal(changes, "restore_fixture", None) + .unwrap(); + assert!(journal.applied.is_empty()); + assert_eq!(journal.changes.len(), 1); + } + + #[test] + fn filesystem_completion_alone_does_not_admit_a_managed_restore_or_foreign_ack() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work"); + std::fs::create_dir(&root).unwrap(); + let store = Store::open( + &temp.path().join("data"), + &root.to_string_lossy(), + super::super::Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + let id = "restore_fixture"; + let dir = storage::private_child(&store.journals, id).unwrap(); + // This is a storage-state fixture, not runtime authority to acknowledge a process. + let origin = SnapshotOrigin { + version: 1, + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + activity_id: "op_fixture".into(), + worktree_id: None, + binding: SnapshotBinding { + directory: store.root.clone(), + root_identity: store.root_id.clone(), + namespace_identity: storage::dir_identity(&store.journals).unwrap(), + journal_id: id.into(), + journal_identity: storage::dir_identity(&dir).unwrap(), + }, + }; + Store::record( + &dir, + &Journal { + version: 2, + location: store.root.clone(), + root: store.root_id.clone(), + created: now(), + completed: true, + changes: vec![], + applied: vec![], + origin: Some(origin.clone()), + acknowledged: false, + }, + ) + .unwrap(); + assert!(store.recovery().unwrap()[0].filesystem_complete); + let refusal = store.preview("unused", "unused", "unused", None); + assert!( + matches!(refusal, Err(RestoreError::Failed(message)) if message.contains("requires recovery")) + ); + let mut foreign = origin.clone(); + foreign.activity_id = "op_foreign".into(); + assert!(store.acknowledge_worker(Some(&foreign)).is_err()); + assert!(store.ensure_recovered().is_err()); + store.acknowledge_worker(Some(&origin)).unwrap(); + assert!(store.ensure_recovered().is_ok()); + assert!(store.journal(id).unwrap().1.acknowledged); + } +} diff --git a/crates/cyber-snapshot/src/copy/storage.rs b/crates/cyber-snapshot/src/copy/storage.rs index 98dd0bdce..f9969d115 100644 --- a/crates/cyber-snapshot/src/copy/storage.rs +++ b/crates/cyber-snapshot/src/copy/storage.rs @@ -7,6 +7,8 @@ use std::os::unix::fs::MetadataExt; use std::path::Path; pub type Result = std::result::Result; +/// Three u64 decimal fields, nanoseconds (at most nine digits) and separators. +pub const IDENTITY_MAX_BYTES: usize = 20 * 3 + 9 + 3; pub fn fail(error: impl std::fmt::Display) -> String { error.to_string() } @@ -16,13 +18,17 @@ pub fn identity(file: &File) -> Result { .created() .map_err(|_| "Filesystem does not expose stable directory creation identity")?; let birth = birth.duration_since(std::time::UNIX_EPOCH).map_err(fail)?; - Ok(format!( + let value = format!( "{}:{}:{}:{}", m.dev(), m.ino(), birth.as_secs(), birth.subsec_nanos() - )) + ); + if value.len() > IDENTITY_MAX_BYTES { + return Err("Snapshot directory identity exceeds its encoded bound".into()); + } + Ok(value) } pub fn dir_identity(dir: &Dir) -> Result { identity(&dir.try_clone().map_err(fail)?.into_std_file()) diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 929abcab2..366f7bd51 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -200,6 +200,72 @@ impl GitSnapshots { } impl Snapshots for GitSnapshots { + fn owned_restore_available(&self, directory: &str) -> bool { + cfg!(any(target_os = "macos", target_os = "linux")) && self.non_git(directory) + } + + fn restore_owned( + &self, + request: cyber_server::runtime::SnapshotRestoreRequest, + worker: cyber_server::runtime::SnapshotWorker, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = (request, worker); + Err(RestoreError::Failed( + "Owned snapshot restore is unavailable on this host".into(), + )) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + if !self.non_git(&request.directory) { + return Err(RestoreError::Failed( + "Owned file-copy restore is unavailable for this Location".into(), + )); + } + let config = (self.config)(Path::new(&request.directory)); + let (ignore, max_file) = Self::options_value(&config); + let max_total = config + .pointer("/snapshots/fallback_max_bytes") + .and_then(Value::as_u64) + .unwrap_or(200 * 1024 * 1024); + let options = copy::Options { + ignore, + max_file: max_file.min(max_total).min(64 * 1024 * 1024), + max_total, + private_roots: self.private_roots.clone(), + }; + let data = self.data_dir.clone(); + let lock = cyber_server::workspace::mutation_lock(Path::new(&request.directory)) + .map_err(|error| RestoreError::Failed(error.to_string()))?; + let workspace = lock.lock_owned().await; + worker + .run(move |work| { + let store = copy::Store::open(&data, &request.directory, options) + .map_err(RestoreError::Failed)?; + let paths = store.restore_worker( + &request.target, + &request.recorded, + &request.expected, + request.paths.as_deref(), + &|| Ok(()), + Some(work), + )?; + let origin = work.origin().cloned(); + Ok(cyber_server::runtime::SnapshotNativeResult::new( + paths, + Box::new(copy::WorkerGuard { + store, + workspace, + origin, + }), + )) + }) + .await + } + }) + } fn recovery(&self, directory: &str) -> BoxFuture<'_, Result, String>> { let directory = directory.to_owned(); Box::pin(async move { GitSnapshots::recovery(self, &directory).await }) diff --git a/crates/cyber-snapshot/tests/file_copy.rs b/crates/cyber-snapshot/tests/file_copy.rs index 5448c9687..4aa95f0dd 100644 --- a/crates/cyber-snapshot/tests/file_copy.rs +++ b/crates/cyber-snapshot/tests/file_copy.rs @@ -45,6 +45,52 @@ impl Project { self.snapshots.track(&self.dir()).await.unwrap().unwrap() } } + +#[tokio::test] +async fn replaced_location_after_plan_never_supplies_a_new_restore_target() { + refuses_replaced_location(false).await; + refuses_replaced_location(true).await; +} + +async fn refuses_replaced_location(symlink: bool) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let p = Project::new(); + p.write("a.txt", "before\n"); + let before = p.track().await; + p.write("a.txt", "after\n"); + let after = p.track().await; + let root = p.root.clone(); + let displaced = root.with_file_name("displaced-root"); + let moved = displaced.clone(); + let calls = AtomicUsize::new(0); + let result = p + .snapshots + .restore_reviewed( + &p.dir(), + &before.tree, + &after.tree, + &after.tree, + Arc::new(move || { + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + std::fs::rename(&root, &moved).unwrap(); + if symlink { + std::os::unix::fs::symlink(&moved, &root).unwrap(); + } else { + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("a.txt"), "after\n").unwrap(); + } + } + Ok(()) + }), + ) + .await; + assert!( + matches!(result, Err(RestoreError::Failed(message)) if message.contains("Location identity changed")) + ); + assert_eq!(std::fs::read(p.root.join("a.txt")).unwrap(), b"after\n"); + assert_eq!(std::fs::read(displaced.join("a.txt")).unwrap(), b"after\n"); +} + #[tokio::test] async fn non_git_snapshots_hash_bytes_preserve_owner_edits_and_restore_without_a_repository() { let p = Project::new(); diff --git a/crates/cyber-store/migrations/20261106000000_snapshot_worker_ownership.sql b/crates/cyber-store/migrations/20261106000000_snapshot_worker_ownership.sql new file mode 100644 index 000000000..e2be0fd3e --- /dev/null +++ b/crates/cyber-store/migrations/20261106000000_snapshot_worker_ownership.sql @@ -0,0 +1,4 @@ +-- Compatibility marker: older binaries do not understand snapshot-worker +-- admission fences or origin-bound acknowledgements. The existing unknown- +-- migration gate must keep those binaries read-only after this upgrade. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index 13e86cb25..88760a756 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -173,6 +173,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261030000000_usage_limit_wait", sql: include_str!("../migrations/20261030000000_usage_limit_wait.sql"), }, + Migration { + id: "20261106000000_snapshot_worker_ownership", + sql: include_str!("../migrations/20261106000000_snapshot_worker_ownership.sql"), + }, ]; pub(crate) enum Outcome { @@ -261,6 +265,58 @@ mod tests { assert_eq!(ids, sorted); } + #[test] + fn snapshot_worker_marker_survives_reopen_and_refuses_the_older_catalog() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("compatibility.db"); + let at = MIGRATIONS + .iter() + .position(|m| m.id == "20261106000000_snapshot_worker_ownership") + .unwrap(); + let legacy = &MIGRATIONS[..at]; + let mut conn = Connection::open(&path).unwrap(); + assert!(matches!( + migrate(&mut conn, legacy).unwrap(), + Outcome::Current + )); + let evidence = r#"{"session_id":"ses_retained","status":"unknown"}"#; + conn.execute("INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('evt_retained','op_retained',0,'native.activity.changed.1',?1,1)", [evidence]).unwrap(); + let tables: i64 = conn + .query_row( + "SELECT count(*) FROM sqlite_schema WHERE type='table'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(matches!( + migrate(&mut conn, MIGRATIONS).unwrap(), + Outcome::Current + )); + assert_eq!( + conn.query_row( + "SELECT count(*) FROM sqlite_schema WHERE type='table'", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + tables + ); + drop(conn); + let mut reopened = Connection::open(path).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut reopened, legacy).unwrap() else { + panic!("older binaries accepted snapshot-worker storage for mutation"); + }; + assert_eq!(unknown, ["20261106000000_snapshot_worker_ownership"]); + let retained: String = reopened + .query_row( + "SELECT data FROM event WHERE id='evt_retained'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(retained, evidence); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/docs/rewind.md b/docs/rewind.md index 46b7cfd27..a9b866908 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -188,7 +188,17 @@ inference. `filesystem_complete` means only that the worker flushed its final filesystem record; it does not acknowledge the Session/native receipt. `applied` contains durable checkpoints; a missing entry is not proof of no effect. -Managed keep/rollback resolution and acknowledgement/reclamation of retained -journals are not implemented yet. Do not edit records or remove evidence to force -a retry. This increment preserves recovery material and provides inspection; it -does not claim complete operator recovery or arbitrary external-writer atomicity. +Reviewed non-Git restores retain their actual worker ownership after a client +disconnects. New input is refused while that worker is active or has unresolved +effects; Stop and read-only inspection remain available between source changes. +On success, Session changes and the exact worker acknowledgement commit together, +followed by acknowledgement of the retained journal. A completed filesystem +record without that acknowledgement still blocks another restore. Legacy journals +remain readable without acquiring new authority to acknowledge native effects. +The snapshot-worker compatibility migration makes older binaries open this +database read-only; they cannot safely enforce the new admission fences. + +Managed keep/rollback resolution, lost-reply acknowledgement reconciliation and +reclamation of retained journals are not implemented yet. Do not edit records or +remove evidence to force a retry. Complete operator recovery and broader native +crash/platform acceptance remain unfinished. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index df51717fe..8b590ad3e 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -403,3 +403,68 @@ No HTTP/SDK/CLI contract or database schema is changed yet. Exact snapshot-worke origin, worker lock retention, new journals, owner recovery operations and safe reclamation remain required before M2–M8 or the parent recovery tasks can close. No recovery success or native acknowledgement is inferred from this prerequisite. + +## Owned non-Git restore worker checkpoint + +Reviewed non-Git code rewind, stage and clear now use runtime-issued synchronous +snapshot workers. The actual blocking callback retains its lifecycle, checkout, +workspace and private-store owners after requester disposal. A separate admission +owner permits inspection and Stop between source effects; each mutation takes the +Session projection lock and rechecks captured authority and revision. New prompt +admission refuses active or durable unresolved snapshot work before hooks and at +consuming admission; exact input retries remain readable. Reload restores the +typed worker fence. Generic and binary scopes cannot adopt snapshot origins. + +Version-2 file-copy intents bind Session creation, activity, checkout, canonical +Location, native root and journal namespace/directory identities. Successful +restoration uses the paired writer to commit Session events and only its own +native acknowledgement, then acknowledges the journal under retained native locks. +Unacknowledged managed completion blocks reuse. Version-1 journals retain their +previous inspection semantics and never gain native settlement authority. The +semantic compatibility migration `20261106000000_snapshot_worker_ownership` adds +no tables, retains existing evidence and makes older migration catalogs refuse +mutation instead of ignoring typed fences. + +The substantive code-review pass corrected holding the Session mutex across the +whole callback, early admission/hook bypass, treating an unobserved origin append +as effect authority, root rebinding, non-UTF-8 origin serialization, downgrade +compatibility and insufficient quota padding. Restore now budgets the full escaped +intent, all progress paths and acknowledgement metadata plus an atomic publication +copy and stage/preimage payloads before creating an intent. A fixture that fits +the former changes-only estimate is refused, while its larger-cap control passes. +Root replacement and symlink-to-displaced-root controls leave both roots unchanged. + +Final macOS snapshot tests passed 39 cases (four unit, 18 file-copy and 17 Git). +The unsupported-platform integration binary selected zero on macOS. Store passed +58 cases including the new migration downgrade/reopen test. Affected server, +snapshot and Store all-target Clippy with warnings denied passed after the final +quota and migration changes. Earlier in this checkpoint, workspace all-target +Clippy passed; 12 admission-authority, three callback-authority, 13 Location +activity, four rewind-preview and six binary-document tests passed. The native +root identity test passed. OpenAPI consistency and SDK generation checks passed; +there is no network DTO change. Strict OpenSpec 1.13.2 passed 117 items; lint +reported zero errors/25 warnings and inventory remained 53 capabilities/1030 +requirements with no ROADMAP change. Formatting and diff checks passed. +All seven runtime snapshot-worker tests passed on the final quota/migration code: +normal exact settlement, requester disposal/Stop/shutdown/reload, pre-effect +refusal, changed revision, rejected consuming commit, ignored failed origin append +and a non-UTF-8 metadata binding. The last is a serialization fixture, not positive +native support for non-UTF-8 filesystem paths. +All 12 real CLI branch/recovery cases passed serialized on that final code, +including owned non-Git stage/clear receipts and preservation of later owner edits. +The CLI build emitted native macOS sandbox deprecation warnings and a Darwin +linker unwind-size warning; these were observed, not classified against main. + +This is partial M2/M3 infrastructure. It does not implement per-change native +source/stage/parent ownership evidence, managed keep/rollback review/confirmation, +request lookup/reconciliation or reclamation. Worker binding and matching bytes +are insufficient rollback proof. Git and legacy restore entrypoints are not yet +migrated to owned workers. Instrumented callback disposal/revision tests are not +real process-kill proof for the new typed restore path; existing killed legacy +file-copy tests do not substitute for that acceptance. Linux/Windows execution, +expanded native crash/race coverage and the SDK TypeScript suite were not run in +this checkpoint. No external model provider was used; CLI fixtures use a scripted +HTTP model. M2–M8 and parent recovery tasks remain open; neither issue is complete. +Crash reconciliation must include reservation-before-origin and +origin-before-intent cuts; this checkpoint conservatively retains their fences +and does not provide their owner recovery operation yet. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index ede9ca85c..5a96b0c29 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -35,6 +35,10 @@ namespace identity and version. The runtime marks the activity as a snapshot filesystem-only worker before it can launch. Snapshot callbacks cannot label a generic shell/process activity as filesystem-only. +The snapshot-worker compatibility migration records the new event semantics even +without new tables. Older migration catalogs must refuse mutation rather than +ignore typed worker fences. Existing native evidence remains unchanged. + A trusted synchronous worker owns the actual mutation lock, private cross-process store lock and checkout/lifecycle resource through its last filesystem call, including when its async requester disappears. Reuse the binary worker's retained diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 33a8d1fe2..3012c57d4 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -85,6 +85,35 @@ - **THEN** its file tree remains readable and inspection reports unavailable exclusion history - **AND** a Location-bound record from a different Location is refused even when its objects are available in the same shadow repository +### Requirement: Runtime-owned snapshot workers +(P1) Managed snapshot restoration SHALL use a runtime-issued synchronous worker owner binding Session creation, native activity, checkout, Location and private journal identities. The actual worker SHALL retain lifecycle, checkout, workspace and private-store ownership through its last filesystem operation and consuming commit even if the requesting task is disposed. Session changes SHALL be excluded from each source mutation and revalidated before the final commit. Ordinary prompt admission SHALL refuse active or unresolved managed snapshot work while exact retries remain inspectable. A successful restore SHALL atomically commit its Session events and exact worker acknowledgement, then acknowledge its journal under retained native proof. A journal alone SHALL NOT settle a generic process, binary document, foreign worker or legacy unknown activity. + +#### Scenario: Request disappears during a live worker +- **WHEN** a requester is disposed after a managed worker records its immutable origin +- **THEN** its actual native owners remain held until the synchronous worker exits, Stop and read-only Session inspection remain available between source effects, and shutdown waits for actual ownership settlement +- **AND** uncertain effects retain their origin and admission fence across runtime reload without replay + +#### Scenario: Session changes before a source effect +- **WHEN** the Session revision changes after preparation but before the next source mutation +- **THEN** that mutation is refused and existing owner bytes remain unchanged + +#### Scenario: Receipt cannot acknowledge another process +- **WHEN** a managed snapshot completes while an unrelated native process has unknown outcome +- **THEN** only the exact snapshot activity can be acknowledged and the unrelated process retains its own unknown evidence and fence + +#### Scenario: Managed journal completion needs writer acknowledgement +- **WHEN** a managed journal records filesystem completion but its runtime commit or journal acknowledgement is unobserved +- **THEN** the retained evidence blocks another restore and is not treated as successful recovery or eligible for deletion + +#### Scenario: Legacy journal compatibility +- **WHEN** inspection reads an origin-less version-1 file-copy intent +- **THEN** it remains readable with its original completion semantics and is never upgraded into authority to acknowledge an unknown native activity + +#### Scenario: Journal quota includes its complete durable record +- **WHEN** restore reserves storage before publishing an intent +- **THEN** the bound includes worker metadata, escaped paths, complete progress and acknowledgement fields, atomic record replacement and retained stage/preimage payloads +- **AND** exceeding the bound refuses before creating the intent or mutating source files + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md index f851207e1..5c0af2f82 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md @@ -23,3 +23,8 @@ #### Scenario: Bounded decision refuses - **WHEN** a checked pair has duplicate, empty or oversized owner IDs, an invalid expected sequence, too many events or an oversized encoding - **THEN** it returns an explicit nonfatal refusal and commits no decision or event changes + +#### Scenario: Older binary cannot ignore new snapshot fences +- **WHEN** a database is upgraded for snapshot-worker ownership +- **THEN** migration `20261106000000_snapshot_worker_ownership` persists without rewriting legacy evidence or adding tables +- **AND** a binary whose migration catalog predates that marker opens the database read-only through the existing unknown-migration gate instead of writing while ignoring new native admission fences diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 1cfee72f6..1a1e57f11 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -48,9 +48,26 @@ exact native proof and actual restart/retry acceptance. Implementation source map: +The non-Git reviewed-confirm path now uses a runtime-issued synchronous worker, +origin-bound version-2 intent and paired Session/native commit followed by journal +acknowledgement. M2–M8 remain open: Git and legacy entry-point integration, +managed keep/rollback and request lookup, resolution receipts, lost-reply journal +reconciliation, cleanup and complete native/platform acceptance are still required. + +Before rollback admission, M3 must retain exact per-change source/staged file and +parent identities, including creation evidence for any new parents. Version-2 +worker origin binds the activity and journal, but file content hashes/mode alone +cannot prove that a current same-byte replacement still belongs to that restore. +Do not infer per-file rollback ownership from the worker binding or from missing +progress. This prerequisite remains open with the complete native crash cuts. +Crash acceptance must also cover reservation before origin publication and origin +publication before intent publication. Those durable fences cannot be force-cleared +or resolved by inventing a missing journal; implement exact pre-effect ownership +proof/reconciliation for these cuts before claiming complete managed recovery. + - `cyber-server/src/runtime/activity.rs` owns native activity records; `binary_documents.rs` retains a real synchronous worker's resources and `binary_recovery.rs` demonstrates document-specific recovery proof. Snapshot workers require their own origin/proof rather than calling binary recovery. - `cyber-server/src/runtime/host.rs::Snapshots` currently exposes restore and read-only recovery inspection; `rewind_preview.rs` owns reviewed confirmation. A dedicated owner recovery admission must work while ordinary admission remains fenced. -- `cyber-snapshot/src/copy/restore.rs` has version-1 intents without native activity origin; `copy/storage.rs` owns private descriptor-relative I/O. `restore.rs` and `git_history.rs` cover Git restore and historical exclusions. Keep historical evidence readable as the new protocol is added. +- `cyber-snapshot/src/copy/restore.rs` reads legacy version-1 intents and writes worker-bound version-2 intents for the owned path; `copy/storage.rs` owns private descriptor-relative I/O. `restore.rs` and `git_history.rs` cover Git restore and historical exclusions. Keep historical evidence readable as the new protocol is added. - `cyber-store/src/store.rs::append_checked` currently appends one aggregate. Separate independent appends cannot establish atomic acknowledgement plus resolution ownership. ## 4. Rules and context inspection From 52147d8dd1ae83a13e6704a7f8b58a949d5403bb Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 13:40:55 -0400 Subject: [PATCH 03/87] Record native ownership before snapshot file and parent effects --- crates/cyber-cli/tests/branches_cli.rs | 7 +- crates/cyber-core/src/binary/native.rs | 18 + crates/cyber-snapshot/src/copy/mod.rs | 8 + crates/cyber-snapshot/src/copy/ownership.rs | 507 +++++++++++++++ .../src/copy/ownership_tests.rs | 610 ++++++++++++++++++ crates/cyber-snapshot/src/copy/restore.rs | 173 ++++- crates/cyber-snapshot/src/copy/storage.rs | 20 +- crates/cyber-snapshot/src/copy/tree.rs | 58 +- crates/cyber-snapshot/tests/file_copy.rs | 19 +- docs/goal-delivery.md | 7 + docs/rewind.md | 10 + .../implementation.md | 65 ++ .../managed-recovery.md | 17 + .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 18 +- 15 files changed, 1460 insertions(+), 83 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/ownership.rs create mode 100644 crates/cyber-snapshot/src/copy/ownership_tests.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 5825b6120..edc56cf8d 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -824,7 +824,12 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { .unwrap(), ) .unwrap(); - assert_eq!(intent["version"], 2); + assert_eq!(intent["version"], 3); + assert_eq!(intent["ownership"]["version"], 1); + assert_eq!( + intent["ownership"]["files"].as_array().unwrap().len(), + intent["changes"].as_array().unwrap().len() + ); assert_eq!(intent["completed"], true); assert_eq!(intent["acknowledged"], true); let origin = &intent["origin"]; diff --git a/crates/cyber-core/src/binary/native.rs b/crates/cyber-core/src/binary/native.rs index 97fc1afdd..2dda791f7 100644 --- a/crates/cyber-core/src/binary/native.rs +++ b/crates/cyber-core/src/binary/native.rs @@ -108,6 +108,20 @@ pub fn validate_private_directory_acl(file: &File) -> Result<(), Error> { /// ACLs, extended attributes and platform flags require separate retained metadata. pub fn plain_snapshot_metadata(file: &File) -> Result { let value = metadata(file)?; + plain_snapshot(file, value) +} + +/// Ordinary metadata for privately prepared snapshot parent directories. Existing +/// owner directories need only routing identity and are never recreated from this. +pub fn plain_snapshot_directory(file: &File) -> Result { + let m = file.metadata().map_err(|_| Error::Io)?; + if !m.is_dir() || m.mode() & 0o7000 != 0 || m.permissions().readonly() { + return Err(Error::File); + } + plain_snapshot(file, metadata_fields(file, &m)?) +} + +fn plain_snapshot(file: &File, value: Metadata) -> Result { if !attributes(file)?.is_empty() { return Err(Error::File); } @@ -126,6 +140,10 @@ pub fn metadata(file: &File) -> Result { if !m.is_file() || m.nlink() != 1 || m.mode() & 0o7000 != 0 || m.permissions().readonly() { return Err(Error::File); } + metadata_fields(file, &m) +} + +fn metadata_fields(file: &File, m: &std::fs::Metadata) -> Result { let mut hash = Sha256::new(); hash.update(m.mode().to_le_bytes()); hash.update(m.uid().to_le_bytes()); diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index a7b065bf1..2c251238b 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,4 +1,5 @@ //! Private BLAKE3 snapshots for exact non-Git Locations; no user repository required. +mod ownership; mod restore; mod storage; mod tree; @@ -216,6 +217,13 @@ impl Store { for name in storage::names(&self.journals, ENTRY_LIMIT)? { let journal = self.journals.open_dir_nofollow(&name).map_err(fail)?; for file in storage::names(&journal, ENTRY_LIMIT * 3)? { + if journal.symlink_metadata(&file).map_err(fail)?.is_dir() { + let parent = journal.open_dir_nofollow(&file).map_err(fail)?; + if !file.starts_with("parent_") || !storage::names(&parent, 1)?.is_empty() { + return Err("Unexpected snapshot journal directory".into()); + } + continue; + } size = size .checked_add( storage::open(&journal, Path::new(&file))? diff --git a/crates/cyber-snapshot/src/copy/ownership.rs b/crates/cyber-snapshot/src/copy/ownership.rs new file mode 100644 index 000000000..0346c9874 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/ownership.rs @@ -0,0 +1,507 @@ +//! Durable native evidence for source effects, independent of progress records. +use super::restore::Change; +use super::storage::{self, Result, fail}; +use super::{ENTRY_LIMIT, Entry, Store}; +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::path::{Component, Path}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Ownership { + version: u32, + parents: Vec, + files: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Parent { + path: String, + identity: String, + stage: Option, + metadata: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct File { + source: Option, + stage: Option, +} + +impl Parent { + fn validate(&self, index: usize, path: &str, identities: &mut BTreeSet) -> Result<()> { + if self.path != path + || !valid_identity(&self.identity) + || !identities.insert(self.identity.clone()) + || self.metadata.is_some() != self.stage.is_some() + || self + .metadata + .as_ref() + .is_some_and(|m| m.mode != 0o700 || !storage::valid_digest(&m.evidence)) + || self + .stage + .as_ref() + .is_some_and(|s| s != &format!("parent_{index}")) + { + return Err("Invalid restore parent evidence".into()); + } + Ok(()) + } + fn verify(&self, dir: &Dir) -> Result<()> { + if storage::dir_identity(dir)? != self.identity { + return Err("Restore parent identity changed; journal retained".into()); + } + if let Some(metadata) = &self.metadata + && storage::parent_metadata(dir)? != *metadata + { + return Err("Restore created-parent metadata changed; journal retained".into()); + } + Ok(()) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub(super) enum Effect { + Unapplied, + Applied, +} + +fn parent_paths(changes: &[Change]) -> Result> { + let mut paths = BTreeSet::new(); + let mut files = BTreeSet::new(); + for change in changes { + super::tree::safe_path(&change.path)?; + if !files.insert(change.path.clone()) || change.before == change.after { + return Err("Invalid restore change inventory".into()); + } + let mut path = Path::new(&change.path).parent(); + while let Some(parent) = path.filter(|p| !p.as_os_str().is_empty()) { + paths.insert(parent.to_str().ok_or("Invalid restore parent")?.to_owned()); + if paths.len() > ENTRY_LIMIT { + return Err("Restore parent count exceeds limit".into()); + } + path = parent.parent(); + } + } + if paths.iter().any(|path| files.contains(path)) { + return Err("Restore file/parent paths overlap".into()); + } + Ok(paths) +} + +fn valid_identity(value: &str) -> bool { + if value.is_empty() || value.len() > storage::IDENTITY_MAX_BYTES { + return false; + } + let parts: Vec<_> = value.split(':').collect(); + parts.len() == 4 + && parts + .iter() + .all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) + && parts[..3].iter().all(|p| p.parse::().is_ok()) + && parts[3].parse::().is_ok_and(|n| n < 1_000_000_000) +} + +fn existing_parent(store: &Store, path: &str) -> Result> { + let mut dir = store.work.try_clone().map_err(fail)?; + for component in Path::new(path).components() { + let Component::Normal(name) = component else { + return Err("Invalid restore parent".into()); + }; + match dir.symlink_metadata(name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(fail(error)), + Ok(_) => dir = dir.open_dir_nofollow(name).map_err(fail)?, + } + } + Ok(Some(dir)) +} + +fn same_filesystem(dir: &Dir, private: &Dir) -> Result<()> { + use std::os::unix::fs::MetadataExt; + let source = dir.try_clone().map_err(fail)?.into_std_file(); + let private = private.try_clone().map_err(fail)?.into_std_file(); + if source.metadata().map_err(fail)?.dev() != private.metadata().map_err(fail)?.dev() { + return Err("Restore requires same-filesystem parents".into()); + } + Ok(()) +} + +fn observed( + store: &Store, + dir: Option<&Dir>, + name: &std::ffi::OsStr, +) -> Result> { + let Some(dir) = dir else { return Ok(None) }; + match dir.symlink_metadata(name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(fail(error)), + Ok(_) => {} + } + // Keep the first inode pinned through hashing and the second route lookup. + let pin = storage::open(dir, Path::new(name))?; + let identity = storage::identity(&pin)?; + let entry = store.inspect_entry(dir, name)?; + let current = storage::open(dir, Path::new(name))?; + if storage::identity(¤t)? != identity { + return Err("Restore file identity changed while inspecting".into()); + } + Ok(Some((identity, entry))) +} + +fn matches( + observed: &Option<(String, Entry)>, + identity: &Option, + entry: &Option, +) -> bool { + match (observed, identity, entry) { + (None, None, None) => true, + (Some((id, value)), Some(want), Some(expected)) => id == want && value == expected, + _ => false, + } +} + +fn valid_entry(entry: &Entry) -> bool { + storage::valid_digest(&entry.object) + && entry.length <= 64 * 1024 * 1024 + && entry.mode & !0o777 == 0 + && entry.mode & 0o222 != 0 +} + +impl File { + fn prepare( + store: &Store, + journal: &Dir, + parent: Option<&Dir>, + index: usize, + change: &Change, + ) -> Result { + let leaf = Path::new(&change.path) + .file_name() + .ok_or("Invalid restore leaf")?; + let source = observed(store, parent, leaf)?; + let staged = observed( + store, + Some(journal), + std::ffi::OsStr::new(&format!("stage_{index}")), + )?; + if source.as_ref().map(|(_, entry)| entry) != change.before.as_ref() + || staged.as_ref().map(|(_, entry)| entry) != change.after.as_ref() + { + return Err("Restore source/stage changed before ownership preparation".into()); + } + if let (Some((identity, _)), Some(parent)) = (&source, parent) { + let pin = storage::open(parent, Path::new(leaf))?; + if storage::identity(&pin)? != *identity { + return Err("Restore original inode changed".into()); + } + // Make the actual displaced preimage durable before any source rename. + // Read-only inspection deliberately never performs this preparation. + storage::sync_file(&pin)?; + } + Ok(Self { + source: source.map(|(id, _)| id), + stage: staged.map(|(id, _)| id), + }) + } +} + +impl Ownership { + pub(super) fn verify_original(&self, index: usize, file: &std::fs::File) -> Result<()> { + if self.files.get(index).and_then(|file| file.source.as_ref()) + != Some(&storage::identity(file)?) + { + return Err("Restore original inode changed".into()); + } + Ok(()) + } + /// Maximum encoded shape, before any private stages or source parents exist. + pub(super) fn budget(changes: &[Change]) -> Result { + let identity = "0".repeat(storage::IDENTITY_MAX_BYTES); + let parents = parent_paths(changes)? + .into_iter() + .enumerate() + .map(|(index, path)| Parent { + path, + identity: identity.clone(), + stage: Some(format!("parent_{index}")), + metadata: Some(cyber_core::binary::native::Metadata { + mode: u32::MAX, + uid: u32::MAX, + gid: u32::MAX, + evidence: "0".repeat(64), + }), + }) + .collect(); + let files = changes + .iter() + .map(|change| File { + source: change.before.as_ref().map(|_| identity.clone()), + stage: change.after.as_ref().map(|_| identity.clone()), + }) + .collect(); + Ok(Self { + version: 1, + parents, + files, + }) + } + + pub(super) fn prepare(store: &Store, journal: &Dir, changes: &[Change]) -> Result { + store.verify_root()?; + same_filesystem(&store.work, journal)?; + for change in changes { + store.check_source_path(&change.path)?; + } + let mut proof = Self::budget(changes)?; + for parent in &mut proof.parents { + let dir = if let Some(dir) = existing_parent(store, &parent.path)? { + parent.stage = None; + parent.metadata = None; + dir + } else { + storage::private_child( + journal, + parent.stage.as_deref().ok_or("Missing parent stage")?, + )? + }; + same_filesystem(&dir, journal)?; + parent.identity = storage::dir_identity(&dir)?; + if parent.stage.is_some() { + parent.metadata = Some(storage::parent_metadata(&dir)?); + storage::sync(&dir)?; + } + } + for (index, change) in changes.iter().enumerate() { + let parent = proof.parent(store, journal, &change.path)?; + proof.files[index] = File::prepare(store, journal, parent.as_ref(), index, change)?; + } + storage::sync(journal)?; + store.verify_root()?; + proof.validate(changes)?; + if proof + .parents + .iter() + .any(|parent| parent.identity == store.root_id) + { + return Err("Restore parent aliases the Location root".into()); + } + Ok(proof) + } + + pub(super) fn validate(&self, changes: &[Change]) -> Result<()> { + let paths = parent_paths(changes)?; + if self.version != 1 + || self.files.len() != changes.len() + || self.parents.len() != paths.len() + { + return Err("Invalid restore ownership inventory".into()); + } + let mut identities = BTreeSet::new(); + for (index, (parent, path)) in self.parents.iter().zip(paths).enumerate() { + parent.validate(index, &path, &mut identities)?; + } + for (file, change) in self.files.iter().zip(changes) { + for (identity, entry) in [(&file.source, &change.before), (&file.stage, &change.after)] + { + if identity.is_some() != entry.is_some() + || entry.as_ref().is_some_and(|entry| !valid_entry(entry)) + || identity.as_ref().is_some_and(|id| !valid_identity(id)) + || identity + .as_ref() + .is_some_and(|id| !identities.insert(id.clone())) + { + return Err("Invalid restore file evidence".into()); + } + } + } + Ok(()) + } + + pub(super) fn parent(&self, store: &Store, journal: &Dir, path: &str) -> Result> { + self.resolve_parent(store, journal, path, None) + } + + pub(super) fn create_parents( + &self, + store: &Store, + journal: &Dir, + path: &str, + authorize: &dyn Fn() -> Result<()>, + ) -> Result { + self.resolve_parent(store, journal, path, Some(authorize))? + .ok_or("Missing restore parent".into()) + } + + fn resolve_parent( + &self, + store: &Store, + journal: &Dir, + path: &str, + authorize: Option<&dyn Fn() -> Result<()>>, + ) -> Result> { + store.verify_root()?; + store.check_source_path(path)?; + let mut dir = store.work.try_clone().map_err(fail)?; + let mut relative = std::path::PathBuf::new(); + for component in Path::new(path) + .parent() + .ok_or("Invalid restore parent")? + .components() + { + let Component::Normal(name) = component else { + return Err("Invalid restore parent".into()); + }; + relative.push(name); + let key = relative.to_str().ok_or("Invalid restore parent")?; + let index = self + .parents + .binary_search_by(|parent| parent.path.as_str().cmp(key)) + .map_err(|_| "Unrecorded restore parent")?; + let proof = &self.parents[index]; + if proof.identity == store.root_id { + return Err("Restore parent aliases the Location root".into()); + } + match dir.symlink_metadata(name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let Some(authorize) = authorize else { + return Ok(None); + }; + authorize()?; + let fresh = self + .parent(store, journal, key)? + .ok_or("Restore parent disappeared")?; + if storage::dir_identity(&fresh)? != storage::dir_identity(&dir)? { + return Err("Restore parent route changed".into()); + } + self.publish_parent(journal, &dir, name, proof)?; + } + Err(error) => return Err(fail(error)), + Ok(_) => {} + } + dir = dir.open_dir_nofollow(name).map_err(fail)?; + proof.verify(&dir)?; + } + Ok(Some(dir)) + } + + fn publish_parent( + &self, + journal: &Dir, + parent: &Dir, + name: &std::ffi::OsStr, + proof: &Parent, + ) -> Result<()> { + let stage = proof + .stage + .as_deref() + .ok_or("Previously present restore parent is missing")?; + let staged = journal.open_dir_nofollow(stage).map_err(fail)?; + proof.verify(&staged)?; + if !storage::names(&staged, 1)?.is_empty() { + return Err("Restore parent stage changed".into()); + } + rustix::fs::renameat_with( + journal, + stage, + parent, + name, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(fail)?; + storage::sync(parent)?; + storage::sync(journal) + } + + pub(super) fn inspect_change( + &self, + store: &Store, + journal: &Dir, + index: usize, + change: &Change, + ) -> Result { + let file = self + .files + .get(index) + .ok_or("Missing restore file evidence")?; + let parent = self.parent(store, journal, &change.path)?; + let leaf = Path::new(&change.path) + .file_name() + .ok_or("Invalid restore leaf")?; + let source = observed(store, parent.as_ref(), leaf)?; + let staged = observed( + store, + Some(journal), + std::ffi::OsStr::new(&format!("stage_{index}")), + )?; + if matches(&source, &file.source, &change.before) + && matches(&staged, &file.stage, &change.after) + { + return Ok(Effect::Unapplied); + } + if matches(&source, &file.stage, &change.after) + && matches(&staged, &file.source, &change.before) + { + return Ok(Effect::Applied); + } + Err(format!( + "Restore native ownership conflicts: {}", + change.path + )) + } + + pub(super) fn inspect( + &self, + store: &Store, + journal: &Dir, + changes: &[Change], + ) -> Result> { + self.validate(changes)?; + store.verify_root()?; + for change in changes { + store.check_source_path(&change.path)?; + } + // Every prepared parent must remain in exactly one recorded slot, even + // when no file below it was published before interruption. + for parent in &self.parents { + let live = existing_parent(store, &parent.path)?; + if let Some(dir) = &live { + parent.verify(dir)?; + } + let live_id = live.as_ref().map(storage::dir_identity).transpose()?; + let staged_id = parent + .stage + .as_ref() + .map(|stage| match journal.symlink_metadata(stage) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(fail(error)), + Ok(_) => { + let dir = journal.open_dir_nofollow(stage).map_err(fail)?; + parent.verify(&dir)?; + if !storage::names(&dir, 1)?.is_empty() { + return Err("Restore parent stage changed".into()); + } + storage::dir_identity(&dir).map(Some) + } + }) + .transpose()? + .flatten(); + if !((live_id.as_ref() == Some(&parent.identity) && staged_id.is_none()) + || (parent.stage.is_some() + && live_id.is_none() + && staged_id.as_ref() == Some(&parent.identity))) + { + return Err("Restore parent ownership conflicts".into()); + } + } + changes + .iter() + .enumerate() + .map(|(index, change)| self.inspect_change(store, journal, index, change)) + .collect() + } +} diff --git a/crates/cyber-snapshot/src/copy/ownership_tests.rs b/crates/cyber-snapshot/src/copy/ownership_tests.rs new file mode 100644 index 000000000..35e64d055 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/ownership_tests.rs @@ -0,0 +1,610 @@ +//! Real file/parent namespace operations; no runtime acknowledgement is asserted. +use super::*; +use cap_std::fs::Dir; +use std::path::PathBuf; + +struct Prepared { + _temp: tempfile::TempDir, + root: PathBuf, + store: Store, + dir: Dir, + record: Journal, +} + +impl Prepared { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work"); + std::fs::create_dir_all(root.join("existing")).unwrap(); + std::fs::write(root.join("existing/replace"), b"before").unwrap(); + std::fs::write(root.join("existing/delete"), b"deleted").unwrap(); + let store = Store::open( + &temp.path().join("data"), + &root.to_string_lossy(), + super::super::Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + let original = store.current("existing/replace").unwrap().unwrap(); + let mut after = store.object(b"after").unwrap(); + after.uid = original.uid; + after.gid = original.gid; + let changes = vec![ + Change { + path: "existing/replace".into(), + before: Some(original), + after: Some(after.clone()), + }, + Change { + path: "existing/delete".into(), + before: store.current("existing/delete").unwrap(), + after: None, + }, + Change { + path: "new/deep/file".into(), + before: None, + after: Some(after), + }, + ]; + let mut record = store + .prepare_journal(changes, "restore_fixture", None) + .unwrap(); + let dir = storage::private_child(&store.journals, "restore_fixture").unwrap(); + Store::record(&dir, &record).unwrap(); + for (index, change) in record.changes.iter().enumerate() { + if let Some(after) = &change.after { + let file = storage::create( + &dir, + &format!("stage_{index}"), + &store.bytes(after).unwrap(), + ) + .unwrap(); + Store::stage_metadata(&file, after).unwrap(); + } + } + record.ownership = Some(Ownership::prepare(&store, &dir, &record.changes).unwrap()); + Store::record(&dir, &record).unwrap(); + Self { + _temp: temp, + root, + store, + dir, + record, + } + } + + fn proof(&self) -> &Ownership { + self.record.ownership.as_ref().unwrap() + } + + fn inspect(&self) -> Result> { + self.proof() + .inspect(&self.store, &self.dir, &self.record.changes) + } + + fn apply(&self, index: usize) { + self.store + .install( + &self.dir, + index, + &self.record.changes[index], + self.proof(), + &|| Ok(()), + ) + .unwrap(); + } + + fn replace_same_bytes(&self, path: &Path, bytes: &[u8]) { + let foreign = self.root.join("foreign"); + std::fs::write(&foreign, bytes).unwrap(); + std::fs::set_permissions(&foreign, std::fs::Permissions::from_mode(0o644)).unwrap(); + std::fs::rename(foreign, path).unwrap(); + } + + fn private_path(&self, name: &str) -> PathBuf { + let key = storage::digest( + format!("{}\0{}", self.store.root.display(), self.store.root_id).as_bytes(), + ); + self.store + .data_path + .join("snapshot/global") + .join(key) + .join("journals/restore_fixture") + .join(name) + } +} + +#[test] +fn missing_progress_does_not_hide_replace_delete_or_create_after_reopen() { + let mut p = Prepared::new(); + assert_eq!( + p.inspect().unwrap(), + [Effect::Unapplied, Effect::Unapplied, Effect::Unapplied] + ); + for index in 0..3 { + p.apply(index); + } + assert!(p.record.applied.is_empty()); + // Reload the durable prepared record, without ever publishing progress. + p.record = p.store.journal("restore_fixture").unwrap().1; + assert!(!p.record.completed); + assert!(p.record.applied.is_empty()); + assert_eq!( + p.inspect().unwrap(), + [Effect::Applied, Effect::Applied, Effect::Applied] + ); + assert_eq!(storage::read(&p.dir, "stage_0", 65536).unwrap(), b"before"); + assert_eq!(storage::read(&p.dir, "stage_1", 65536).unwrap(), b"deleted"); + assert_eq!( + std::fs::read(p.root.join("new/deep/file")).unwrap(), + b"after" + ); + let data = p.store.data_path.clone(); + let options = p.store.options.clone(); + drop(p.store); + p.store = Store::open(&data, &p.root.to_string_lossy(), options).unwrap(); + assert_eq!( + p.inspect().unwrap(), + [Effect::Applied, Effect::Applied, Effect::Applied] + ); + assert!(p.store.ensure_recovered().is_err()); +} + +#[test] +fn same_byte_foreign_source_is_refused_before_and_after_replacement() { + for applied in [false, true] { + let p = Prepared::new(); + let bytes = if applied { + p.apply(0); + b"after".as_slice() + } else { + b"before".as_slice() + }; + let path = p.root.join("existing/replace"); + p.replace_same_bytes(&path, bytes); + assert_eq!( + p.store + .inspect_entry( + &p.store.parent("existing/replace").unwrap(), + std::ffi::OsStr::new("replace") + ) + .unwrap(), + if applied { + p.record.changes[0].after.clone().unwrap() + } else { + p.record.changes[0].before.clone().unwrap() + } + ); + assert!(p.inspect().is_err()); + assert_eq!(std::fs::read(path).unwrap(), bytes); + assert_eq!( + std::fs::read(p.root.join("existing/delete")).unwrap(), + b"deleted" + ); + } +} + +#[test] +fn same_byte_foreign_preimage_and_stage_are_not_owned() { + for applied in [false, true] { + let p = Prepared::new(); + if applied { + p.apply(0); + } + let path = p.private_path("stage_0"); + let bytes = if applied { + b"before".as_slice() + } else { + b"after".as_slice() + }; + p.replace_same_bytes(&path, bytes); + assert!(p.inspect().is_err()); + assert_eq!(std::fs::read(path).unwrap(), bytes); + } +} + +#[test] +fn foreign_parent_and_symlink_to_displaced_parent_are_refused() { + for symlink in [false, true] { + let p = Prepared::new(); + let old = p.root.join("old-parent"); + std::fs::rename(p.root.join("existing"), &old).unwrap(); + if symlink { + std::os::unix::fs::symlink(&old, p.root.join("existing")).unwrap(); + } else { + std::fs::create_dir(p.root.join("existing")).unwrap(); + std::fs::copy(old.join("replace"), p.root.join("existing/replace")).unwrap(); + std::fs::copy(old.join("delete"), p.root.join("existing/delete")).unwrap(); + } + assert!(p.inspect().is_err()); + assert!( + p.store + .install(&p.dir, 0, &p.record.changes[0], p.proof(), &|| Ok(())) + .is_err() + ); + assert_eq!(std::fs::read(old.join("replace")).unwrap(), b"before"); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"before" + ); + } +} + +#[test] +fn durable_parent_preparation_explains_a_cut_before_file_publication() { + let p = Prepared::new(); + let parent = p + .proof() + .create_parents(&p.store, &p.dir, "new/deep/file", &|| Ok(())) + .unwrap(); + assert!(!p.root.join("new/deep/file").exists()); + assert_eq!( + p.inspect().unwrap(), + [Effect::Unapplied, Effect::Unapplied, Effect::Unapplied] + ); + assert_eq!( + storage::dir_identity(&parent).unwrap(), + storage::dir_identity(&p.store.parent("new/deep/file").unwrap()).unwrap() + ); + // Equal empty directories do not replace the recorded created parents. + std::fs::rename(p.root.join("new/deep"), p.root.join("displaced-deep")).unwrap(); + std::fs::create_dir(p.root.join("new/deep")).unwrap(); + assert!(p.inspect().is_err()); +} + +#[test] +fn inspection_of_changed_owner_bytes_never_populates_snapshot_objects() { + let p = Prepared::new(); + let before = storage::names(&p.store.objects, 100).unwrap(); + std::fs::write(p.root.join("existing/replace"), b"new owner content").unwrap(); + assert!(p.inspect().is_err()); + assert_eq!(storage::names(&p.store.objects, 100).unwrap(), before); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"new owner content" + ); +} + +#[test] +fn aliases_in_source_or_private_stage_refuse_native_ownership() { + for stage in [false, true] { + let p = Prepared::new(); + let path = if stage { + p.private_path("stage_0") + } else { + p.root.join("existing/replace") + }; + std::fs::hard_link(path, p.root.join("alias")).unwrap(); + assert!(p.inspect().is_err()); + assert!( + p.store + .install(&p.dir, 0, &p.record.changes[0], p.proof(), &|| Ok(())) + .is_err() + ); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"before" + ); + } +} + +#[test] +fn malformed_prepared_inventory_does_not_acquire_effect_authority() { + let p = Prepared::new(); + let original = serde_json::to_value(p.proof()).unwrap(); + let mut cases = vec![]; + let mut bad = original.clone(); + bad["version"] = 0.into(); + cases.push(bad); + let mut bad = original.clone(); + bad["files"][0]["source"] = "0:0:0:1000000000".into(); + cases.push(bad); + let mut bad = original.clone(); + bad["files"][0]["source"] = original["files"][0]["stage"].clone(); + cases.push(bad); + let mut bad = original.clone(); + bad["parents"][0]["path"] = "../foreign".into(); + cases.push(bad); + let mut bad = original; + bad["parents"][1]["stage"] = "other".into(); + cases.push(bad); + for case in cases { + let proof: Ownership = serde_json::from_value(case).unwrap(); + assert!(proof.validate(&p.record.changes).is_err()); + } + let mut changes = p.record.changes.clone(); + changes[0].before.as_mut().unwrap().object = "invalid-object".into(); + assert!(p.proof().validate(&changes).is_err()); +} + +#[test] +fn foreign_or_populated_parent_stages_refuse_before_publication() { + for foreign in [false, true] { + let p = Prepared::new(); + let path = p.private_path("parent_1"); + if foreign { + std::fs::rename(&path, p.private_path("foreign-parent")).unwrap(); + std::fs::create_dir(&path).unwrap(); + } else { + std::fs::write(path.join("owner-file"), b"retain owner data").unwrap(); + } + assert!(p.inspect().is_err()); + assert!( + p.store + .install(&p.dir, 2, &p.record.changes[2], p.proof(), &|| Ok(())) + .is_err() + ); + assert!(!p.root.join("new").exists()); + } +} + +#[test] +fn revocation_is_rechecked_before_each_parent_and_file_publication() { + for before_file in [false, true] { + let p = Prepared::new(); + let stop_path = p.root.join(if before_file { "new/deep" } else { "new" }); + let authorize = || { + if stop_path.exists() { + Err("admission revoked".into()) + } else { + Ok(()) + } + }; + assert!( + p.store + .install(&p.dir, 2, &p.record.changes[2], p.proof(), &authorize) + .is_err() + ); + assert!(p.root.join("new").is_dir()); + assert_eq!(p.root.join("new/deep").is_dir(), before_file); + assert!(!p.root.join("new/deep/file").exists()); + assert_eq!( + p.inspect().unwrap(), + [Effect::Unapplied, Effect::Unapplied, Effect::Unapplied] + ); + } + let p = Prepared::new(); + assert!( + p.store + .install(&p.dir, 0, &p.record.changes[0], p.proof(), &|| Err( + "admission revoked".into() + )) + .is_err() + ); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"before" + ); +} + +#[test] +fn created_parent_permission_changes_refuse_while_existing_parent_modes_are_preserved() { + for published in [false, true] { + let p = Prepared::new(); + let path = if published { + p.proof() + .create_parents(&p.store, &p.dir, "new/deep/file", &|| Ok(())) + .unwrap(); + p.root.join("new") + } else { + p.private_path("parent_1") + }; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(p.inspect().is_err()); + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } + let p = Prepared::new(); + std::fs::set_permissions( + p.root.join("existing"), + std::fs::Permissions::from_mode(0o700), + ) + .unwrap(); + assert!(p.inspect().is_ok()); + p.apply(0); + assert_eq!( + std::fs::metadata(p.root.join("existing")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); +} + +#[test] +fn added_created_parent_attributes_refuse_without_discarding_metadata() { + let p = Prepared::new(); + p.proof() + .create_parents(&p.store, &p.dir, "new/deep/file", &|| Ok(())) + .unwrap(); + let dir = std::fs::File::open(p.root.join("new")).unwrap(); + #[cfg(target_os = "linux")] + let name = c"user.cyber_owner"; + #[cfg(target_os = "macos")] + let name = c"com.cybercode.owner"; + rustix::fs::fsetxattr( + &dir, + name, + b"owner metadata", + rustix::fs::XattrFlags::empty(), + ) + .unwrap(); + assert!(p.inspect().is_err()); + let mut bytes = [0u8; 64]; + let size = rustix::fs::fgetxattr(&dir, name, bytes.as_mut_slice()).unwrap(); + assert_eq!(&bytes[..size], b"owner metadata"); + assert!(!p.root.join("new/deep/file").exists()); +} + +#[test] +fn unprepared_and_legacy_records_remain_readable_without_native_file_proof() { + let mut p = Prepared::new(); + p.record.ownership = None; + for version in [1, 3] { + p.record.version = version; + Store::record(&p.dir, &p.record).unwrap(); + let (_, record) = p.store.journal("restore_fixture").unwrap(); + assert!(p.store.verify_applied(&p.dir, &record).is_err()); + assert!(p.store.ensure_recovered().is_err()); + } + p.record.completed = true; + Store::record(&p.dir, &p.record).unwrap(); + assert!(p.store.journal("restore_fixture").is_err()); + p.record.completed = false; + p.record.applied.push("wrong-progress".into()); + Store::record(&p.dir, &p.record).unwrap(); + assert!(p.store.journal("restore_fixture").is_err()); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"before" + ); +} + +#[test] +fn native_ownership_cut_worker() { + let Some(base) = std::env::var_os("CYBER_OWNERSHIP_CUT_BASE") else { + return; + }; + let base = PathBuf::from(base); + let cut: usize = std::env::var("CYBER_OWNERSHIP_CUT_PHASE") + .unwrap() + .parse() + .unwrap(); + let store = Store::open( + &base.join("data"), + &base.join("work").to_string_lossy(), + super::super::Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + let (dir, record) = store.journal("restore_fixture").unwrap(); + let ownership = record.ownership.as_ref().unwrap(); + if cut >= 1 { + ownership + .create_parents(&store, &dir, "new/deep/file", &|| Ok(())) + .unwrap(); + } + for index in 0..cut.saturating_sub(1) { + store + .install(&dir, index, &record.changes[index], ownership, &|| Ok(())) + .unwrap(); + } + // No progress record is written. Parent observes the actual namespace before + // killing this process, then opens the storage through a fresh owner. + std::fs::write(base.join("cut-ready"), cut.to_string()).unwrap(); + loop { + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +struct ChildGuard(std::process::Child); +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[test] +fn process_kill_at_parent_and_file_cuts_recovers_ownership_without_progress_or_replay() { + for cut in 0..5 { + verify_killed_cut(cut); + } +} + +fn verify_killed_cut(cut: usize) { + let mut p = Prepared::new(); + let base = p._temp.path().to_owned(); + drop(p.store); + drop(p.dir); + let log = std::fs::File::create(base.join("worker.log")).unwrap(); + let mut child = ChildGuard( + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "copy::restore::ownership_tests::native_ownership_cut_worker", + "--nocapture", + ]) + .env("CYBER_OWNERSHIP_CUT_BASE", &base) + .env("CYBER_OWNERSHIP_CUT_PHASE", cut.to_string()) + .env("PATH", "") + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(), + ); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while !base.join("cut-ready").exists() { + assert!( + std::time::Instant::now() < deadline && child.0.try_wait().unwrap().is_none(), + "native worker missed cut {cut}: {}", + std::fs::read_to_string(base.join("worker.log")).unwrap() + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert_live_cut(&p.root, cut); + child.0.kill().unwrap(); + child.0.wait().unwrap(); + p.store = Store::open( + &base.join("data"), + &p.root.to_string_lossy(), + super::super::Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + (p.dir, p.record) = p.store.journal("restore_fixture").unwrap(); + assert_recovered_cut(&p, cut); +} + +fn assert_live_cut(root: &Path, cut: usize) { + // Independently observe the real child effect; its marker alone is not proof. + assert_eq!(root.join("new/deep").exists(), cut >= 1); + assert_eq!( + std::fs::read(root.join("existing/replace")).unwrap(), + if cut >= 2 { + b"after".as_slice() + } else { + b"before".as_slice() + } + ); + assert_eq!(root.join("existing/delete").exists(), cut < 3); + assert_eq!(root.join("new/deep/file").exists(), cut >= 4); +} + +fn assert_recovered_cut(p: &Prepared, cut: usize) { + assert!(p.record.applied.is_empty()); + assert!(!p.record.completed); + let effects = p.inspect().unwrap(); + for (index, effect) in effects.iter().enumerate() { + assert_eq!( + *effect, + if cut >= index + 2 { + Effect::Applied + } else { + Effect::Unapplied + } + ); + } + assert!(p.store.ensure_recovered().is_err()); + if cut >= 2 { + assert_eq!(storage::read(&p.dir, "stage_0", 65536).unwrap(), b"before"); + } + if cut >= 3 { + assert_eq!(storage::read(&p.dir, "stage_1", 65536).unwrap(), b"deleted"); + } +} diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 2923b5b7c..1f315813b 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -1,4 +1,5 @@ //! A durable intent precedes every namespace mutation; incomplete intents fence reuse. +use super::ownership::{Effect, Ownership}; use super::storage::{self, Result, fail}; use super::{ENTRY_LIMIT, Entry, RECORD_LIMIT, Store, Tree, now}; use cap_fs_ext::DirExt; @@ -11,16 +12,16 @@ use std::path::Path; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct Change { - path: String, - before: Option, - after: Option, +pub(super) struct Change { + pub(super) path: String, + pub(super) before: Option, + pub(super) after: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Journal { - // Version 2 binds the worker, not historical per-file rollback identity. - // Resolution must not substitute Entry hashes for exact native change proof. + // Version 3 adds prepared native file/parent ownership. Version 2 binds + // only the worker and cannot acquire that historical proof retroactively. version: u32, location: std::path::PathBuf, root: String, @@ -32,13 +33,15 @@ struct Journal { origin: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] acknowledged: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + ownership: Option, } impl Store { pub(super) fn ensure_recovered(&self) -> Result<()> { for id in storage::names(&self.journals, ENTRY_LIMIT)? { let (_, record) = self.journal(&id)?; - if !record.completed || (record.version == 2 && !record.acknowledged) { + if !record.completed || (record.origin.is_some() && !record.acknowledged) { return Err( "Snapshot restore requires recovery; private journal and preimages retained" .into(), @@ -55,14 +58,28 @@ impl Store { .map_err(fail)?; if !matches!( (record.version, record.origin.is_some()), - (1, false) | (2, true) + (1, false) | (2, true) | (3, _) ) || record.location != self.root || record.root != self.root_id || record.changes.len() > ENTRY_LIMIT - || (record.version == 1 && record.acknowledged) + || record.applied.len() > record.changes.len() + || !record + .applied + .iter() + .zip(&record.changes) + .all(|(path, change)| path == &change.path) + || (record.completed && record.applied.len() != record.changes.len()) + || (record.origin.is_none() && record.acknowledged) + || (record.version < 3 && record.ownership.is_some()) + || (record.version == 3 + && record.ownership.is_none() + && (record.completed || !record.applied.is_empty())) { return Err("Invalid snapshot recovery journal".into()); } + if let Some(ownership) = &record.ownership { + ownership.validate(&record.changes)?; + } if let Some(origin) = &record.origin { let binding = &origin.binding; if origin.version != 1 @@ -293,6 +310,9 @@ impl Store { } } storage::sync(&dir).map_err(RestoreError::Failed)?; + journal.ownership = + Some(Ownership::prepare(self, &dir, &journal.changes).map_err(RestoreError::Failed)?); + Self::record(&dir, &journal).map_err(RestoreError::Failed)?; for i in 0..journal.changes.len() { authorize().map_err(RestoreError::Failed)?; if let Some(worker) = worker.as_ref() { @@ -304,17 +324,37 @@ impl Store { .as_ref() .ok_or("Missing snapshot worker origin")?, )?; - self.install(&dir, i, &journal.changes[i]) + self.install( + &dir, + i, + &journal.changes[i], + journal + .ownership + .as_ref() + .ok_or("Missing restore ownership")?, + &|| worker.authorize(), + ) }) .map_err(RestoreError::Failed)?; } else { - self.install(&dir, i, &journal.changes[i]) - .map_err(RestoreError::Failed)?; + self.install( + &dir, + i, + &journal.changes[i], + journal + .ownership + .as_ref() + .ok_or_else(|| RestoreError::Failed("Missing restore ownership".into()))?, + authorize, + ) + .map_err(RestoreError::Failed)?; } journal.applied.push(journal.changes[i].path.clone()); Self::record(&dir, &journal).map_err(RestoreError::Failed)?; } self.verify_root().map_err(RestoreError::Failed)?; + self.verify_applied(&dir, &journal) + .map_err(RestoreError::Failed)?; journal.completed = true; Self::record(&dir, &journal).map_err(RestoreError::Failed)?; Ok(journal.applied) @@ -338,8 +378,9 @@ impl Store { })) }) .transpose()?; + let ownership = Some(Ownership::budget(&changes)?); let mut journal = Journal { - version: if origin.is_some() { 2 } else { 1 }, + version: 3, location: self.root.clone(), root: self.root_id.clone(), created: now(), @@ -350,6 +391,7 @@ impl Store { changes, origin, acknowledged: worker.is_some(), + ownership, }; let bytes = serde_json::to_vec(&journal).map_err(fail)?.len() as u64; if bytes > RECORD_LIMIT { @@ -373,6 +415,7 @@ impl Store { journal.applied.clear(); journal.origin = None; journal.acknowledged = false; + journal.ownership = None; Ok(journal) } @@ -380,10 +423,26 @@ impl Store { self.verify_root()?; let Some(origin) = origin else { return Ok(()) }; self.verify_worker_namespace(origin)?; - let (_, record) = self.journal(&origin.binding.journal_id)?; + let (dir, record) = self.journal(&origin.binding.journal_id)?; if record.origin.as_ref() != Some(origin) || !record.completed { return Err("Snapshot worker completion differs from its retained intent".into()); } + self.verify_applied(&dir, &record)?; + Ok(()) + } + + fn verify_applied(&self, dir: &cap_std::fs::Dir, record: &Journal) -> Result<()> { + let ownership = record + .ownership + .as_ref() + .ok_or("Restore has no prepared native ownership")?; + if ownership + .inspect(self, dir, &record.changes)? + .iter() + .any(|effect| *effect != Effect::Applied) + { + return Err("Restore filesystem completion differs from native ownership".into()); + } Ok(()) } @@ -432,7 +491,7 @@ impl Store { .map_err(fail)?; file.set_permissions(std::fs::Permissions::from_mode(entry.mode)) .map_err(fail)?; - file.sync_all().map_err(fail) + storage::sync_file(file) } fn record(dir: &cap_std::fs::Dir, journal: &Journal) -> Result<()> { let bytes = serde_json::to_vec(journal).map_err(fail)?; @@ -441,22 +500,29 @@ impl Store { } storage::atomic(dir, "intent.json", &bytes) } - fn install(&self, dir: &cap_std::fs::Dir, index: usize, change: &Change) -> Result<()> { + fn install( + &self, + dir: &cap_std::fs::Dir, + index: usize, + change: &Change, + ownership: &Ownership, + authorize: &dyn Fn() -> Result<()>, + ) -> Result<()> { self.verify_root()?; - if self.current(&change.path)? != change.before { + if ownership.inspect_change(self, dir, index, change)? != Effect::Unapplied { return Err(format!( "Restore source changed; journal retained: {}", change.path )); } - let parent = self.parent(&change.path, true)?; + let parent = ownership.create_parents(self, dir, &change.path, authorize)?; let parent_id = storage::dir_identity(&parent)?; let leaf = Path::new(&change.path) .file_name() .ok_or("Invalid restore leaf")?; let stage = format!("stage_{index}"); match (&change.before, &change.after) { - (Some(before), Some(after)) => { + (Some(before), Some(_)) => { let target = cyber_core::binary::native::Target::capture( &self.root, Path::new(&change.path), @@ -471,15 +537,21 @@ impl Store { return Err("Restore source metadata changed; journal retained".into()); } let (_, original) = target.validate().map_err(fail)?; - let stage_file = storage::open(dir, Path::new(&stage))?; - cyber_core::binary::native::copy_metadata(&original, &stage_file).map_err(fail)?; - Self::stage_metadata(&stage_file, after)?; + ownership.verify_original(index, &original)?; + // Snapshot entries admit only plain ownership/mode. Those fields + // are already durable on the stage before its identity is logged. + // Copying native timestamps here would change macOS birth identity. + if ownership.inspect_change(self, dir, index, change)? != Effect::Unapplied { + return Err("Restore source changed before exchange".into()); + } + self.authorize_restore_effect(dir, ownership, &change.path, &parent_id, authorize)?; cyber_core::binary::native::exchange(dir, &stage, &target).map_err(fail)?; - if self.read_entry(dir, std::ffi::OsStr::new(&stage))? != *before { + if self.inspect_entry(dir, std::ffi::OsStr::new(&stage))? != *before { return Err("Displaced restore preimage changed; recovery required".into()); } } (Some(before), None) => { + self.authorize_restore_effect(dir, ownership, &change.path, &parent_id, authorize)?; rustix::fs::renameat_with( &parent, leaf, @@ -488,33 +560,63 @@ impl Store { rustix::fs::RenameFlags::NOREPLACE, ) .map_err(fail)?; - if self.read_entry(dir, std::ffi::OsStr::new(&stage))? != *before { + if self.inspect_entry(dir, std::ffi::OsStr::new(&stage))? != *before { return Err("Displaced deletion preimage changed; recovery required".into()); } } - (None, Some(_)) => rustix::fs::renameat_with( - dir, - &stage, - &parent, - leaf, - rustix::fs::RenameFlags::NOREPLACE, - ) - .map_err(fail)?, + (None, Some(_)) => { + self.authorize_restore_effect(dir, ownership, &change.path, &parent_id, authorize)?; + rustix::fs::renameat_with( + dir, + &stage, + &parent, + leaf, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(fail)?; + } (None, None) => {} } storage::sync(&parent)?; storage::sync(dir)?; self.verify_root()?; - if storage::dir_identity(&self.parent(&change.path, false)?)? != parent_id { + if storage::dir_identity( + &ownership + .parent(self, dir, &change.path)? + .ok_or("Missing restore parent")?, + )? != parent_id + { return Err("Restore parent identity changed; journal retained".into()); } - if self.current(&change.path)? != change.after { + if ownership.inspect_change(self, dir, index, change)? != Effect::Applied { return Err("Restore result changed; recovery required".into()); } Ok(()) } + + fn authorize_restore_effect( + &self, + dir: &cap_std::fs::Dir, + ownership: &Ownership, + path: &str, + parent_id: &str, + authorize: &dyn Fn() -> Result<()>, + ) -> Result<()> { + authorize()?; + let fresh = ownership + .parent(self, dir, path)? + .ok_or("Restore parent disappeared")?; + if storage::dir_identity(&fresh)? != parent_id { + return Err("Restore parent route changed".into()); + } + Ok(()) + } } +#[cfg(test)] +#[path = "ownership_tests.rs"] +mod ownership_tests; + #[cfg(test)] mod tests { use super::*; @@ -560,7 +662,7 @@ mod tests { .unwrap() .is_empty() ); - store.options.max_total = 20000; + store.options.max_total = 100000; let journal = store .prepare_journal(changes, "restore_fixture", None) .unwrap(); @@ -613,6 +715,7 @@ mod tests { applied: vec![], origin: Some(origin.clone()), acknowledged: false, + ownership: None, }, ) .unwrap(); diff --git a/crates/cyber-snapshot/src/copy/storage.rs b/crates/cyber-snapshot/src/copy/storage.rs index f9969d115..7ab6ed5d7 100644 --- a/crates/cyber-snapshot/src/copy/storage.rs +++ b/crates/cyber-snapshot/src/copy/storage.rs @@ -16,7 +16,7 @@ pub fn identity(file: &File) -> Result { let m = file.metadata().map_err(fail)?; let birth = m .created() - .map_err(|_| "Filesystem does not expose stable directory creation identity")?; + .map_err(|_| "Filesystem does not expose stable native creation identity")?; let birth = birth.duration_since(std::time::UNIX_EPOCH).map_err(fail)?; let value = format!( "{}:{}:{}:{}", @@ -33,7 +33,19 @@ pub fn identity(file: &File) -> Result { pub fn dir_identity(dir: &Dir) -> Result { identity(&dir.try_clone().map_err(fail)?.into_std_file()) } +pub fn sync_file(file: &File) -> Result<()> { + file.sync_all().map_err(fail)?; + #[cfg(target_os = "macos")] + rustix::fs::fcntl_fullfsync(file).map_err(fail)?; + Ok(()) +} pub fn sync(dir: &Dir) -> Result<()> { + readable_directory(dir)?.sync_all().map_err(fail) +} +pub fn parent_metadata(dir: &Dir) -> Result { + cyber_core::binary::native::plain_snapshot_directory(&readable_directory(dir)?).map_err(fail) +} +fn readable_directory(dir: &Dir) -> Result { // cap-std can hold O_PATH directories on Linux, which fsync rejects. // Reopen the pinned directory itself for I/O, without resolving an ambient path. let fd = rustix::fs::openat( @@ -46,7 +58,7 @@ pub fn sync(dir: &Dir) -> Result<()> { rustix::fs::Mode::empty(), ) .map_err(fail)?; - File::from(fd).sync_all().map_err(fail) + Ok(File::from(fd)) } pub fn private_child(parent: &Dir, name: &str) -> Result { child(parent, name, 0o077) @@ -134,9 +146,7 @@ pub fn create(dir: &Dir, name: &str, bytes: &[u8]) -> Result { .follow(FollowSymlinks::No); let mut file = dir.open_with(name, &options).map_err(fail)?.into_std(); file.write_all(bytes).map_err(fail)?; - file.sync_all().map_err(fail)?; - #[cfg(target_os = "macos")] - rustix::fs::fcntl_fullfsync(&file).map_err(fail)?; + sync_file(&file)?; Ok(file) } pub fn atomic(dir: &Dir, name: &str, bytes: &[u8]) -> Result<()> { diff --git a/crates/cyber-snapshot/src/copy/tree.rs b/crates/cyber-snapshot/src/copy/tree.rs index 63080b8a1..23a6e5282 100644 --- a/crates/cyber-snapshot/src/copy/tree.rs +++ b/crates/cyber-snapshot/src/copy/tree.rs @@ -134,6 +134,12 @@ impl Store { Ok(()) } pub(super) fn read_entry(&self, dir: &Dir, name: &std::ffi::OsStr) -> Result { + self.entry(dir, name, true) + } + pub(super) fn inspect_entry(&self, dir: &Dir, name: &std::ffi::OsStr) -> Result { + self.entry(dir, name, false) + } + fn entry(&self, dir: &Dir, name: &std::ffi::OsStr, save: bool) -> Result { let mut file = storage::open(dir, Path::new(name))?; let before = file.metadata().map_err(fail)?; let metadata = cyber_core::binary::native::plain_snapshot_metadata(&file) @@ -161,15 +167,37 @@ impl Store { { return Err("Snapshot source changed while hashing".into()); } - let mut entry = self.object(&bytes)?; + let mut entry = if save { + self.object(&bytes)? + } else { + Entry { + object: storage::digest(&bytes), + length: bytes.len() as u64, + mode: 0, + uid: 0, + gid: 0, + } + }; entry.mode = metadata.mode; entry.uid = metadata.uid; entry.gid = metadata.gid; Ok(entry) } - pub(super) fn parent(&self, path: &str, create: bool) -> Result { - safe_path(path)?; + pub(super) fn parent(&self, path: &str) -> Result { + self.check_source_path(path)?; let parent = Path::new(path).parent().ok_or("Invalid snapshot parent")?; + self.verify_root()?; + let mut dir = self.work.try_clone().map_err(fail)?; + for component in parent.components() { + let Component::Normal(name) = component else { + return Err("Invalid snapshot parent".into()); + }; + dir = dir.open_dir_nofollow(name).map_err(fail)?; + } + Ok(dir) + } + pub(super) fn check_source_path(&self, path: &str) -> Result<()> { + safe_path(path)?; if protected(Path::new(path)) || self.root.join(path).starts_with(&self.data_path) || self @@ -180,30 +208,10 @@ impl Store { { return Err("Protected snapshot path".into()); } - self.verify_root()?; - let mut dir = self.work.try_clone().map_err(fail)?; - for component in parent.components() { - let Component::Normal(name) = component else { - return Err("Invalid snapshot parent".into()); - }; - if create && dir.symlink_metadata(name).is_err() { - use cap_std::fs::{DirBuilder, DirBuilderExt}; - let mut builder = DirBuilder::new(); - builder.mode(0o700); - dir.create_dir_with(name, &builder).map_err(fail)?; - storage::sync(&dir)?; - #[cfg(target_os = "macos")] - cyber_core::binary::native::validate_private_directory_acl( - &dir.open_dir_nofollow(name).map_err(fail)?.into_std_file(), - ) - .map_err(fail)?; - } - dir = dir.open_dir_nofollow(name).map_err(fail)?; - } - Ok(dir) + Ok(()) } pub(super) fn current(&self, path: &str) -> Result> { - let parent = match self.parent(path, false) { + let parent = match self.parent(path) { Ok(parent) => parent, Err(error) => { // Only a genuinely absent ancestor is absence; aliases and errors refuse. diff --git a/crates/cyber-snapshot/tests/file_copy.rs b/crates/cyber-snapshot/tests/file_copy.rs index 4aa95f0dd..9510d5cfb 100644 --- a/crates/cyber-snapshot/tests/file_copy.rs +++ b/crates/cyber-snapshot/tests/file_copy.rs @@ -243,7 +243,6 @@ async fn storage_cap_and_foreign_location_do_not_authorize_restore() { } #[tokio::test] async fn admission_loss_after_a_replacement_retains_journal_and_blocks_replay() { - use std::sync::atomic::{AtomicUsize, Ordering}; let p = Project::new(); p.write("a.txt", "old a"); p.write("b.txt", "old b"); @@ -251,8 +250,7 @@ async fn admission_loss_after_a_replacement_retains_journal_and_blocks_replay() p.write("a.txt", "new a"); p.write("b.txt", "new b"); let after = p.track().await; - let calls = Arc::new(AtomicUsize::new(0)); - let count = calls.clone(); + let first = p.root.join("a.txt"); let result = p .snapshots .restore_reviewed( @@ -261,7 +259,7 @@ async fn admission_loss_after_a_replacement_retains_journal_and_blocks_replay() &after.tree, &after.tree, Arc::new(move || { - if count.fetch_add(1, Ordering::SeqCst) >= 2 { + if std::fs::read(&first).ok().as_deref() == Some(b"old a") { Err("admission revoked".into()) } else { Ok(()) @@ -395,7 +393,7 @@ fn killed_restore_worker() { let ids: Value = serde_json::from_slice(&std::fs::read(base.join("ids.json")).unwrap()).unwrap(); let marker = base.join("restore-paused"); - let count = std::sync::atomic::AtomicUsize::new(0); + let first = base.join("work/a.txt"); let _ = snapshots .restore_reviewed( &directory, @@ -403,7 +401,7 @@ fn killed_restore_worker() { ids[1].as_str().unwrap(), ids[1].as_str().unwrap(), Arc::new(move || { - if count.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 2 { + if std::fs::read(&first).ok().as_deref() == Some(b"before a") { std::fs::write(&marker, b"first native replacement acknowledged").unwrap(); loop { std::thread::sleep(std::time::Duration::from_millis(20)); @@ -561,7 +559,7 @@ async fn macos_acl_grants_cannot_make_private_snapshot_storage_readable() { #[tokio::test] async fn recovery_inspection_does_not_wait_for_a_paused_native_writer() { - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicBool, Ordering}; struct Release(Arc); impl Drop for Release { fn drop(&mut self) { @@ -582,8 +580,9 @@ async fn recovery_inspection_does_not_wait_for_a_paused_native_writer() { let snapshots = p.snapshots.clone(); let directory = p.dir(); let resume = released.clone(); + let first = p.root.join("a.txt"); let task = tokio::spawn(async move { - let count = AtomicUsize::new(0); + let signaled = AtomicBool::new(false); snapshots .restore_reviewed( &directory, @@ -591,7 +590,9 @@ async fn recovery_inspection_does_not_wait_for_a_paused_native_writer() { &after.tree, &after.tree, Arc::new(move || { - if count.fetch_add(1, Ordering::SeqCst) == 2 { + if std::fs::read(&first).ok().as_deref() == Some(b"old a") + && !signaled.swap(true, Ordering::SeqCst) + { signal.notify_one(); while !resume.load(Ordering::SeqCst) { std::thread::sleep(std::time::Duration::from_millis(10)); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index d1e1082b9..076cf5bb6 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -141,3 +141,10 @@ for the required design review; that protocol is not implemented or approved by publication alone. Issue #93 was automatically closed by #110 despite remaining scope and has been reopened. The owner's closed #94 is unchanged. Keep the existing one-PR delivery constraint and separate Plugins/hooks policy gates. + +On 2026-11-06 the user approved the concrete managed-recovery protocol and requested +implementation. Its worker and file-copy journal prerequisites are in progress; +keep/rollback, reconciliation, Git journaling, reclamation and full native/client +acceptance remain required. Approval and current evidence are recorded in the daily +coding change's tasks.md and implementation.md; publishing a prerequisite does not +complete recovery or either issue. diff --git a/docs/rewind.md b/docs/rewind.md index a9b866908..af22d1fe5 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -198,6 +198,16 @@ remain readable without acquiring new authority to acknowledge native effects. The snapshot-worker compatibility migration makes older binaries open this database read-only; they cannot safely enforce the new admission fences. +New version-3 file-copy journals record original/staged file identities and parent +identities before source changes. Missing parents are prepared privately and their +identities recorded before publication. This evidence distinguishes a completed +rename from a later same-byte owner replacement even when progress was not saved. +The journal bound includes this metadata; at most 4096 distinct parent paths are +prepared for one restore. Older intents do not gain historical file ownership +evidence from matching content. Read-only native inspection never captures new +objects or changes owner files. Created-parent permissions and other metadata are +checked against preparation; existing owner-directory modes are preserved. + Managed keep/rollback resolution, lost-reply acknowledgement reconciliation and reclamation of retained journals are not implemented yet. Do not edit records or remove evidence to force a retry. Complete operator recovery and broader native diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 8b590ad3e..8a43bcf25 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -468,3 +468,68 @@ HTTP model. M2–M8 and parent recovery tasks remain open; neither issue is comp Crash reconciliation must include reservation-before-origin and origin-before-intent cuts; this checkpoint conservatively retains their fences and does not provide their owner recovery operation yet. + +## Prepared file-copy ownership and native cut evidence + +Refreshed external state: origin/main remains `db9e171e`, issues #93/#92 are open, +and the repository has no open PRs. This continuation adds the per-file evidence +needed by M3/M4 without claiming the managed resolution operations exist. + +New non-Git file-copy restores write version-3 intents, including the legacy direct +adapter path (which still has no runtime origin). Before a source namespace change, +the intent records native device/inode/birth identities for original and staged +files and every parent route. Missing parents are staged privately; their actual +native identities and ordinary metadata are durable before no-replace publication. +Original inodes, stages and staged parents are flushed before ownership publication. +The full journal quota includes this evidence and a maximum of 4096 distinct +parents. No missing progress entry becomes evidence that a rename did not happen. + +Internal read-only inspection compares both live and private slots against native +identities and expected content/ordinary metadata. It does not populate content +objects, sync owner files, create parents, acknowledge native activity or change +the transcript. Same-byte foreign source/preimage/stage files, aliases, replaced +parents, populated/foreign parent stages and changed created-parent metadata refuse. +Existing owner-directory mode changes are preserved. Source effects recheck current +authority and parent routing before each parent publication and file operation. +Whole-plan recovery review, fingerprints and resolution intents remain to implement. + +Version-1/2 journals stay readable without gaining historical per-file evidence; +unprepared version-3 intents retain their fence without claiming file proof. +Validation rejects malformed/duplicate native inventories and contradictory +progress/completion. No cleanup or legacy native-fence clearing was added. + +The code-review skill pass found that macOS native metadata copying changed the +stage birth timestamp after its identity was recorded. Snapshot files support only +plain ownership/mode, so all supported stage metadata is now prepared before +recording identity and is not recopied during installation. Shared binary file +metadata checks retain their original behavior; a common fingerprint helper also +supports ordinary snapshot-created directories. The pass added original-inode +durability, per-parent revocation checks, same-filesystem preflight, metadata +conflicts and read-only object-inventory controls. Test helpers now pause based on +observed file effects rather than authorization callback counts. Clippy's test +complexity and nested-if findings were corrected without allowances. + +Final local macOS verification: snapshot tests passed 54 cases (19 unit, 18 +file-copy and 17 Git). One unit drives five real child-process kill cuts after +preparation, parent publication and each replace/delete/create, without writing +progress. Parent assertions observe the actual namespace before killing the child; +fresh Store owners then prove the retained native slots without replay. The child +helper's ordinary test invocation performs no cut; its five explicit child +invocations supply that evidence. The unsupported-platform binary selected zero. +Five Core native tests passed, including file/metadata exchange and ACL preservation. +All 12 real CLI branch/recovery tests passed on the final source, preceded by three +focused non-Git cases. Core/snapshot/CLI all-target Clippy with warnings denied, +formatting and diff checks passed. Strict OpenSpec passed 117 items, lint reported +zero errors/25 warnings, and inventory remained 53 capabilities/1030 requirements +without a ROADMAP diff. Native macOS sandbox deprecation and Darwin linker warnings +were observed; their origin was not assigned against main. No external provider +was used; CLI inference fixtures use scripted HTTP responses. + +These process kills prove storage ownership, not typed runtime activity settlement +or keep/rollback acceptance. Git-native equivalence, managed runtime crash cuts, +reservation/origin/intent gaps, resolution/retry/acknowledgement reconciliation, +safe reclamation and owner API/SDK/CLI/TUI recovery remain open. Linux/Windows, +cross-filesystem native runs, the full workspace tests/Clippy, the runtime binary +document suite and SDK TypeScript checks were not rerun in this checkpoint. No +network DTO changed or SDK generation was required. M2–M8 and parent tasks remain +open; neither issue is complete. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 5a96b0c29..b3644478b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -57,6 +57,23 @@ identities are rechecked before every filesystem effect and the final commit. ## Rollback and commit ordering +New file-copy records retain a prepared native ownership section before source +effects: each original/staged file's device, inode and creation identity, every +parent route's native identity, and private staged identities for missing parents. +Missing parents are prepared privately and published with no-replace only after +this section is durable. Inspection compares both source and private slots to the +prepared identities and expected bytes/metadata; progress alone never decides +whether a rename occurred. Same-byte foreign replacements, aliases and changed +parents refuse. Older records without this evidence remain readable but do not +gain rollback authority. A crash during private preparation retains an unprepared +intent; it does not authorize inferring any source replacement. +Original inodes, prepared stages and new parents are flushed before prepared +ownership is published. New-parent ownership includes its ordinary metadata; +changed mode/ownership, ACLs, attributes or flags refuse rather than licensing +directory removal. Existing owner-directory modes are preserved. Authority is +rechecked before each parent publication and file namespace operation, including +when cancellation occurs between parent creation and file publication. + 1. Validate original intent/stage objects and complete all conflict checks under the owned locks. Freeze a bounded reviewed plan and digest with current file and parent identities. A changed fingerprint requires a fresh owner review. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 3012c57d4..256999286 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -114,6 +114,12 @@ - **THEN** the bound includes worker metadata, escaped paths, complete progress and acknowledgement fields, atomic record replacement and retained stage/preimage payloads - **AND** exceeding the bound refuses before creating the intent or mutating source files +#### Scenario: Native ownership survives missing progress +- **WHEN** a file-copy restore stops after a namespace operation but before progress is recorded +- **THEN** its durable prepared section identifies original/staged files and all parents, including privately staged missing parents, so inspection can distinguish an applied effect without replay +- **AND** an equal-byte replacement with a foreign native identity, a changed parent or alias refuses ownership proof; older journals without prepared evidence remain readable without gaining rollback authority +- **AND** original preimages and private preparations are flushed before prepared evidence is published, created-parent metadata changes refuse, and authority is rechecked before each parent publication and file operation + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 1a1e57f11..c7376defd 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -49,17 +49,19 @@ exact native proof and actual restart/retry acceptance. Implementation source map: The non-Git reviewed-confirm path now uses a runtime-issued synchronous worker, -origin-bound version-2 intent and paired Session/native commit followed by journal +origin-bound intent and paired Session/native commit followed by journal acknowledgement. M2–M8 remain open: Git and legacy entry-point integration, managed keep/rollback and request lookup, resolution receipts, lost-reply journal reconciliation, cleanup and complete native/platform acceptance are still required. -Before rollback admission, M3 must retain exact per-change source/staged file and -parent identities, including creation evidence for any new parents. Version-2 -worker origin binds the activity and journal, but file content hashes/mode alone -cannot prove that a current same-byte replacement still belongs to that restore. -Do not infer per-file rollback ownership from the worker binding or from missing -progress. This prerequisite remains open with the complete native crash cuts. +New version-3 file-copy intents retain native per-change source/staged file and +parent identities, including privately staged identities for new parents before +publication. Read-only inspection classifies actual source/private slots without +using progress as proof; five storage-level process-kill cuts verify this after +fresh ownership. Legacy version-1/2 intents have no historical per-file proof. +Do not infer rollback ownership from matching bytes or missing progress. Git +equivalence, managed resolution intents/operations, runtime crash cuts and the +complete native platform matrix remain open with M3/M8. Crash acceptance must also cover reservation before origin publication and origin publication before intent publication. Those durable fences cannot be force-cleared or resolved by inventing a missing journal; implement exact pre-effect ownership @@ -67,7 +69,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - `cyber-server/src/runtime/activity.rs` owns native activity records; `binary_documents.rs` retains a real synchronous worker's resources and `binary_recovery.rs` demonstrates document-specific recovery proof. Snapshot workers require their own origin/proof rather than calling binary recovery. - `cyber-server/src/runtime/host.rs::Snapshots` currently exposes restore and read-only recovery inspection; `rewind_preview.rs` owns reviewed confirmation. A dedicated owner recovery admission must work while ordinary admission remains fenced. -- `cyber-snapshot/src/copy/restore.rs` reads legacy version-1 intents and writes worker-bound version-2 intents for the owned path; `copy/storage.rs` owns private descriptor-relative I/O. `restore.rs` and `git_history.rs` cover Git restore and historical exclusions. Keep historical evidence readable as the new protocol is added. +- `cyber-snapshot/src/copy/restore.rs` reads legacy version-1/2 intents and writes version-3 native ownership records. `copy/ownership.rs` prepares file/parent evidence and inspects actual slots; `copy/storage.rs` owns private descriptor-relative I/O. `restore.rs` and `git_history.rs` cover Git restore and historical exclusions. Keep historical evidence readable as the new protocol is added. - `cyber-store/src/store.rs::append_checked` currently appends one aggregate. Separate independent appends cannot establish atomic acknowledgement plus resolution ownership. ## 4. Rules and context inspection From d90b519ff80961042792523407be7cf1f00252da Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 18:02:30 -0400 Subject: [PATCH 04/87] Add snapshot recovery review and journaled filesystem resolution --- crates/cyber-cli/tests/branches_cli.rs | 52 ++ crates/cyber-server/src/http/openapi.rs | 3 + crates/cyber-server/src/http/sessions.rs | 16 + crates/cyber-server/src/runtime/host.rs | 16 + crates/cyber-server/src/runtime/mod.rs | 7 + .../src/runtime/snapshot_recovery.rs | 257 +++++++++ .../src/runtime/snapshot_recovery_types.rs | 78 +++ .../cyber-server/src/runtime/snapshot_work.rs | 26 +- .../src/runtime/snapshot_work_tests.rs | 42 ++ crates/cyber-snapshot/src/copy/mod.rs | 51 +- crates/cyber-snapshot/src/copy/ownership.rs | 246 +++++++++ .../src/copy/ownership_tests.rs | 4 +- crates/cyber-snapshot/src/copy/resolution.rs | 518 ++++++++++++++++++ .../src/copy/resolution_tests.rs | 468 ++++++++++++++++ crates/cyber-snapshot/src/copy/restore.rs | 204 +++++-- crates/cyber-snapshot/src/copy/storage.rs | 12 +- crates/cyber-snapshot/src/lib.rs | 110 ++++ docs/rewind.md | 32 +- .../implementation.md | 74 +++ .../managed-recovery.md | 15 + .../specs/snapshots-checkpoints/spec.md | 15 + .../complete-daily-coding-surfaces/tasks.md | 20 + sdk/openapi.json | 351 ++++++++++++ sdk/typescript/src/generated/operations.ts | 21 + sdk/typescript/src/generated/types.ts | 69 +++ sdk/typescript/test/rewind.test.ts | 15 + 26 files changed, 2649 insertions(+), 73 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_recovery.rs create mode 100644 crates/cyber-server/src/runtime/snapshot_recovery_types.rs create mode 100644 crates/cyber-snapshot/src/copy/resolution.rs create mode 100644 crates/cyber-snapshot/src/copy/resolution_tests.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index edc56cf8d..3cfbbe02f 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -732,6 +732,58 @@ fn interrupt_and_wait_idle(env: &Env, id: &str) { } } +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn recovery_review_binds_the_session_discloses_acknowledgement_and_starts_no_inference() { + let model = scripted_model::ScriptedModel::start(reply); + let env = Env::new(&model); + std::fs::remove_dir_all(env.root.join("repo/.git")).unwrap(); + let id = env.exec(None, "WRITE-A"); + let first = env.user_ids(&id)[0].clone(); + reviewed_apply( + &env, + &id, + &reviewed_selection(&env, &first, "code", "commit"), + ); + let catalog = json(&env.cyber(&["api", "GET", &format!("/sessions/{id}/rewind/recovery")])); + let journal = catalog["data"][0]["id"].as_str().unwrap(); + let created = json(&env.cyber(&["api", "POST", "/sessions", "-d", "{\"title\":\"foreign\"}"])); + let other = created["data"]["id"].as_str().unwrap().to_owned(); + let requests = model.requests().len(); + let original_messages = env.messages(&id); + for action in ["keep", "rollback"] { + let body = json!({"journal_id":journal,"action":action}).to_string(); + let response = json(&env.cyber(&[ + "api", + "POST", + &format!("/sessions/{id}/rewind/recovery/review"), + "-d", + &body, + ])); + let review = &response["data"]; + assert_eq!(review["origin"]["session_id"], id); + assert_eq!(review["acknowledged"], true); + assert!(review["confirmation"].is_null()); + assert!(!review["conflicts"].as_array().unwrap().is_empty()); + assert!(review["exclusions"].is_array()); + let foreign = env.cyber(&[ + "api", + "POST", + &format!("/sessions/{other}/rewind/recovery/review"), + "-d", + &body, + ]); + assert!(!foreign.status.success()); + assert!( + String::from_utf8_lossy(&foreign.stdout).contains("different Session") + || String::from_utf8_lossy(&foreign.stderr).contains("different Session") + ); + } + assert_eq!(env.file("a.txt"), None); + assert_eq!(env.messages(&id), original_messages); + assert_eq!(model.requests().len(), requests); +} + #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn reviewed_non_git_stage_and_clear_keep_exact_acknowledged_worker_receipts() { diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index ea1875b44..134de48d9 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -760,6 +760,9 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { .ok::>(g), op("get", "/sessions/{sessionID}/rewind/recovery", "v1.session.snapshotRecovery", "Inspect retained filesystem restore journals without acknowledging or replaying native effects") .ok::>>(g), + op("post", "/sessions/{sessionID}/rewind/recovery/review", "v1.session.snapshotRecoveryReview", "Review retained snapshot effects without restoring, acknowledging native activity or starting inference") + .body::(g) + .ok::>(g), op("post", "/sessions/{sessionID}/rewind/preview", "v1.session.rewindPreview", "Review a rewind without changing files or the active conversation") .body::(g) .ok::>(g), diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index 20c20fb05..fef94d88b 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -346,6 +346,10 @@ pub fn routes() -> Router { .route(&format!("{s}/rewind"), post(rewind)) .route(&format!("{s}/rewind/preview"), post(rewind_preview)) .route(&format!("{s}/rewind/recovery"), get(snapshot_recovery)) + .route( + &format!("{s}/rewind/recovery/review"), + post(snapshot_recovery_review), + ) .route(&format!("{s}/rewind/confirm"), post(rewind_confirm)) .route(&format!("{s}/revert/stage"), post(stage)) .route(&format!("{s}/revert/clear"), post(clear)) @@ -1197,6 +1201,18 @@ async fn snapshot_recovery( data: state.runtime.snapshot_recovery(&id).await?, })) } +async fn snapshot_recovery_review( + State(state): State, + Path(id): Path, + Json(selection): Json, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_recovery_review(&id, &selection) + .await?, + })) +} async fn rewind_preview( State(state): State, Path(id): Path, diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 7a0cd6bc2..b239a1eb6 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -1429,6 +1429,22 @@ pub struct SnapshotRecovery { /// Working-tree snapshots for a Location. pub trait Snapshots: Send + Sync { + fn recovery_review( + &self, + _directory: &str, + _journal: &str, + _action: super::SnapshotRecoveryAction, + ) -> BoxFuture<'_, Result> { + Box::pin(async { Err("Snapshot recovery review is unavailable".into()) }) + } + fn resolve_recovery( + &self, + _directory: String, + _request: super::SnapshotRecoveryConfirmation, + _worker: super::SnapshotWorker, + ) -> BoxFuture<'_, Result> { + Box::pin(async { Err(RestoreError::Failed("Owned recovery is unavailable".into())) }) + } /// Only synchronous filesystem hosts retaining runtime-issued worker ownership. fn owned_restore_available(&self, _directory: &str) -> bool { false diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 7829c9fc6..a924a3478 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -122,11 +122,18 @@ mod model; mod requests; mod rewind; mod rewind_preview; +mod snapshot_recovery; +mod snapshot_recovery_types; mod snapshot_work; pub use rewind_preview::{ RewindAction, RewindCodePreview, RewindConfirmation, RewindEffect, RewindPreview, RewindSelection, }; +pub use snapshot_recovery_types::{ + SnapshotRecoveryAction, SnapshotRecoveryConfirmation, SnapshotRecoveryImage, + SnapshotRecoveryPath, SnapshotRecoveryReceipt, SnapshotRecoveryReview, + SnapshotRecoverySelection, +}; pub use snapshot_work::{ SnapshotBinding, SnapshotCompletion, SnapshotNativeGuard, SnapshotNativeResult, SnapshotOrigin, SnapshotRestoreRequest, SnapshotWork, SnapshotWorker, diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs new file mode 100644 index 000000000..04de345e5 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -0,0 +1,257 @@ +//! Owner review is read-only. Native acknowledgement requires the consuming owner transaction. +use super::{Runtime, RuntimeError, SnapshotRecoveryReview, SnapshotRecoverySelection, activity}; +use rusqlite::OptionalExtension; + +impl Runtime { + pub async fn snapshot_recovery_review( + &self, + session: &str, + selection: &SnapshotRecoverySelection, + ) -> Result { + let before = self.session_info(session).await?; + let mut review = self + .inner + .options + .snapshots + .recovery_review(&before.directory, &selection.journal_id, selection.action) + .await + .map_err(RuntimeError::Invalid)?; + if review.journal_id != selection.journal_id || review.action != selection.action { + return Err(RuntimeError::Corrupt( + "Snapshot host returned a different recovery selection".into(), + )); + } + if let Some(origin) = &review.origin { + let directory = std::fs::canonicalize(&before.directory) + .map_err(|error| RuntimeError::Invalid(error.to_string()))?; + if origin.binding.directory != directory { + return Err(RuntimeError::Conflict( + "Recovery Location differs from the current Session".into(), + )); + } + let created = self + .inner + .store + .read_events(session, -1, 1)? + .events + .first() + .map(|event| event.id.clone()); + if origin.session_id != session + || created.as_deref() != Some(&origin.session_created_id) + || origin.worktree_id != before.worktree_id + { + return Err(RuntimeError::Conflict( + "Recovery journal belongs to a different Session/checkout".into(), + )); + } + self.validate_recovery_activity(&mut review)?; + } + let after = self.session_info(session).await?; + if after.directory != before.directory || after.worktree_id != before.worktree_id { + return Err(RuntimeError::Conflict( + "Session Location changed during recovery review".into(), + )); + } + Ok(review) + } + + fn validate_recovery_activity( + &self, + review: &mut SnapshotRecoveryReview, + ) -> Result<(), RuntimeError> { + let origin = review.origin.as_ref().expect("caller checked origin"); + let id = origin.activity_id.clone(); + let rows = self.inner.store.read(move |db| { + let latest = db.query_row("SELECT data FROM event WHERE aggregate_id=?1 AND type='native.activity.changed.1' ORDER BY seq DESC LIMIT 1", [&id], |row| row.get::<_,String>(0)).optional()?; + let launched = db.query_row("SELECT data FROM event WHERE aggregate_id=?1 AND seq=1 AND type='native.activity.changed.1'", [&id], |row| row.get::<_,String>(0)).optional()?; + Ok((latest,launched)) + })?; + let Some((latest, launched)) = rows.0.zip(rows.1) else { + review.confirmation = None; + review + .conflicts + .push("Exact native origin evidence is unavailable".into()); + return Ok(()); + }; + for data in [&latest, &launched] { + let record: activity::Record = serde_json::from_str(data) + .map_err(|error| RuntimeError::Corrupt(error.to_string()))?; + if !record.snapshot_worker + || record.binary_document.is_some() + || record.session_id != origin.session_id + || record.snapshot_origin.as_ref() != Some(origin) + { + return Err(RuntimeError::Conflict( + "Journal cannot acknowledge this native activity".into(), + )); + } + } + let record: activity::Record = serde_json::from_str(&latest) + .map_err(|error| RuntimeError::Corrupt(error.to_string()))?; + if !matches!(record.status.as_str(), "pending" | "unknown" | "settled") { + return Err(RuntimeError::Corrupt( + "Invalid snapshot native activity status".into(), + )); + } + let active = self + .inner + .activities + .lock() + .map_err(|_| RuntimeError::Conflict("Activity registry unavailable".into()))? + .contains_key(&origin.activity_id); + if active || record.status == "settled" { + review.confirmation = None; + review.conflicts.push( + if active { + "Original snapshot worker is still active" + } else { + "Native activity is already acknowledged; reconcile its existing receipt" + } + .into(), + ); + } + Ok(()) + } +} + +#[cfg(all(test, any(target_os = "macos", target_os = "linux")))] +mod tests { + use super::*; + use crate::runtime::{SnapshotBinding, SnapshotOrigin, SnapshotRecoveryAction}; + + fn origin(runtime: &Runtime, session: &str, activity: &str) -> SnapshotOrigin { + SnapshotOrigin { + version: 1, + session_id: session.into(), + session_created_id: runtime + .inner + .store + .read_events(session, -1, 1) + .unwrap() + .events[0] + .id + .clone(), + activity_id: activity.into(), + worktree_id: None, + binding: SnapshotBinding { + directory: std::env::temp_dir(), + root_identity: "instrumented".into(), + namespace_identity: "instrumented".into(), + journal_id: "restore_fixture".into(), + journal_identity: "instrumented".into(), + lock_identity: Some("instrumented_lock".into()), + }, + } + } + fn review(origin: SnapshotOrigin) -> SnapshotRecoveryReview { + SnapshotRecoveryReview { + journal_id: "restore_fixture".into(), + action: SnapshotRecoveryAction::Keep, + origin: Some(origin), + filesystem_complete: false, + acknowledged: false, + confirmation: Some("a".repeat(64)), + paths: vec![], + exclusions: None, + recorded_created_parents: None, + conflicts: vec![], + } + } + #[tokio::test] + async fn exact_origin_review_distinguishes_live_unknown_and_settled_without_writing() { + let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let mut scope = activity::Scope::reserve_snapshot(&runtime.inner, session.clone()) + .await + .unwrap(); + let bound = origin(&runtime, &session, scope.id()); + scope.launch_snapshot(bound.clone()).unwrap(); + let sequence = runtime.inner.store.aggregate_seq(scope.id()).unwrap(); + let mut live = review(bound.clone()); + runtime.validate_recovery_activity(&mut live).unwrap(); + assert!(live.confirmation.is_none()); + assert!(live.conflicts[0].contains("still active")); + assert_eq!( + runtime.inner.store.aggregate_seq(scope.id()).unwrap(), + sequence + ); + drop(scope); + let sequence = runtime + .inner + .store + .aggregate_seq(&bound.activity_id) + .unwrap(); + let mut unknown = review(bound.clone()); + runtime.validate_recovery_activity(&mut unknown).unwrap(); + assert!(unknown.confirmation.is_some()); + assert!(unknown.conflicts.is_empty()); + assert_eq!( + runtime + .inner + .store + .aggregate_seq(&bound.activity_id) + .unwrap(), + sequence + ); + // Use an independently owned typed scope to exercise acknowledged state. + let mut settled = activity::Scope::reserve_snapshot(&runtime.inner, session.clone()) + .await + .unwrap(); + let bound = origin(&runtime, &session, settled.id()); + settled.launch_snapshot(bound.clone()).unwrap(); + settled.settle().unwrap(); + drop(settled); + let mut acknowledged = review(bound); + runtime + .validate_recovery_activity(&mut acknowledged) + .unwrap(); + assert!(acknowledged.confirmation.is_none()); + assert!(acknowledged.conflicts[0].contains("already acknowledged")); + runtime.shutdown().await; + } + #[tokio::test] + async fn journal_cannot_relabel_a_generic_activity_or_replace_a_typed_origin() { + let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut scope = activity::Scope::reserve(&runtime.inner, &handle) + .await + .unwrap(); + scope.launch().unwrap(); + let before = runtime.inner.store.aggregate_seq(scope.id()).unwrap(); + assert!( + runtime + .validate_recovery_activity(&mut review(origin(&runtime, &session, scope.id()))) + .is_err() + ); + assert_eq!( + runtime.inner.store.aggregate_seq(scope.id()).unwrap(), + before + ); + drop(scope); + let mut typed = activity::Scope::reserve_snapshot(&runtime.inner, session.clone()) + .await + .unwrap(); + let mut bound = origin(&runtime, &session, typed.id()); + typed.launch_snapshot(bound.clone()).unwrap(); + bound.binding.journal_identity = "foreign".into(); + assert!( + runtime + .validate_recovery_activity(&mut review(bound)) + .is_err() + ); + drop(typed); + runtime.shutdown().await; + } + #[tokio::test] + async fn missing_native_evidence_disables_confirmation_without_creating_activity() { + let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let mut missing = review(origin(&runtime, &session, "op_missing")); + runtime.validate_recovery_activity(&mut missing).unwrap(); + assert!(missing.confirmation.is_none()); + assert!(!missing.conflicts.is_empty()); + assert_eq!( + runtime.inner.store.aggregate_seq("op_missing").unwrap(), + None + ); + runtime.shutdown().await; + } +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs new file mode 100644 index 000000000..695c74364 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs @@ -0,0 +1,78 @@ +//! Owner recovery contracts. A filesystem outcome alone never acknowledges native work. +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SnapshotRecoveryAction { + Keep, + Rollback, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SnapshotRecoveryImage { + pub hash: String, + pub bytes: u64, + pub mode: u32, + pub uid: u32, + pub gid: u32, + /// Expected images and proposed copies need not identify a live inode. + pub native_identity: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SnapshotRecoveryPath { + pub path: String, + /// `applied`, `unapplied`, or `foreign`; foreign files are never rollback targets. + pub state: String, + pub current: Option, + pub recorded_before: Option, + pub recorded_after: Option, + /// Current private slot: staged target or displaced preimage, without inventing ownership. + pub retained: Option, + pub proposed: Option, + pub changes: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SnapshotRecoveryReview { + pub journal_id: String, + pub action: SnapshotRecoveryAction, + pub origin: Option, + pub filesystem_complete: bool, + pub acknowledged: bool, + /// Absent when conflicts or unavailable historical evidence prohibit confirmation. + pub confirmation: Option, + pub paths: Vec, + /// None means historical exclusions were not retained; not an empty exclusion set. + pub exclusions: Option>, + /// Recorded created parents remain protected until acknowledged safe cleanup. + pub recorded_created_parents: Option>, + pub conflicts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotRecoveryConfirmation { + pub journal_id: String, + pub action: SnapshotRecoveryAction, + pub confirmation: String, + pub request_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotRecoverySelection { + pub journal_id: String, + pub action: SnapshotRecoveryAction, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SnapshotRecoveryReceipt { + pub journal_id: String, + pub resolution_id: String, + pub request_id: String, + pub action: SnapshotRecoveryAction, + pub confirmation: String, + pub original: super::SnapshotOrigin, + pub filesystem_complete: bool, +} diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 385c37455..ec8266fc1 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use tokio::sync::OwnedRwLockReadGuard; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] pub struct SnapshotBinding { pub directory: PathBuf, @@ -17,9 +17,12 @@ pub struct SnapshotBinding { pub namespace_identity: String, pub journal_id: String, pub journal_identity: String, + /// Exact private lock inode. Legacy origins do not acquire this proof retroactively. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lock_identity: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] pub struct SnapshotOrigin { pub version: u32, @@ -47,12 +50,17 @@ pub trait SnapshotNativeGuard: Send { pub struct SnapshotNativeResult { pub paths: Vec, + pub recovery: Option, guard: Box, } impl SnapshotNativeResult { pub fn new(paths: Vec, guard: Box) -> Self { - Self { paths, guard } + Self { + paths, + guard, + recovery: None, + } } } @@ -149,6 +157,10 @@ impl SnapshotWork<'_> { return Err("Snapshot Location path cannot be represented as UTF-8".into()); } if binding.directory != self.owner.directory + || binding + .lock_identity + .as_ref() + .is_some_and(|id| id.is_empty() || id.len() > 256) || [ &binding.root_identity, &binding.namespace_identity, @@ -204,11 +216,19 @@ impl SnapshotWorker { } impl SnapshotCompletion { + pub fn recovery_receipt(&self) -> Option<&super::SnapshotRecoveryReceipt> { + self.native.recovery.as_ref() + } pub(crate) fn commit( mut self, state: &mut SessionState, events: Vec, ) -> Result<(), RuntimeError> { + if self.native.recovery.is_some() { + return Err(RuntimeError::Conflict( + "Recovery requires its exact native consuming commit".into(), + )); + } self.native .guard .verify(self.worker.scope.snapshot_origin()) diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 2f41277c1..c5822f6d0 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -28,6 +28,7 @@ fn binding(directory: PathBuf) -> SnapshotBinding { namespace_identity: "instrumented_namespace".into(), journal_id: "restore_test".into(), journal_identity: "instrumented_journal".into(), + lock_identity: Some("instrumented_lock".into()), } } async fn worker(runtime: &Runtime, session: &str) -> SnapshotWorker { @@ -59,6 +60,47 @@ fn restart(runtime: &Runtime, root: &std::path::Path) -> Runtime { }) } +#[tokio::test] +async fn ordinary_restore_commit_cannot_consume_a_recovery_receipt_or_acknowledge_it() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let acknowledgements = Arc::new(AtomicUsize::new(0)); + let count = acknowledgements.clone(); + let native_binding = binding(temp.path().canonicalize().unwrap()); + let completed = owner + .run(move |work| { + let origin = work.begin(native_binding).unwrap(); + let mut result = SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin.clone()), + acknowledgements: count, + }), + ); + result.recovery = Some(super::super::SnapshotRecoveryReceipt { + journal_id: "restore_original".into(), + resolution_id: "restore_resolution".into(), + request_id: "request".into(), + action: super::super::SnapshotRecoveryAction::Keep, + confirmation: "a".repeat(64), + original: origin, + filesystem_complete: true, + }); + Ok(result) + }) + .await + .unwrap(); + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + let sequence = state.last_seq; + assert!(completed.commit(&mut state, vec![]).is_err()); + assert_eq!(state.last_seq, sequence); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + drop(state); + runtime.shutdown().await; +} + #[tokio::test] async fn non_utf8_location_binding_refuses_before_serializing_a_native_origin() { use std::os::unix::ffi::OsStringExt; diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 2c251238b..28340dfcf 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,5 +1,6 @@ //! Private BLAKE3 snapshots for exact non-Git Locations; no user repository required. mod ownership; +mod resolution; mod restore; mod storage; mod tree; @@ -57,6 +58,7 @@ pub(crate) struct Store { root_id: String, data_path: PathBuf, work: Dir, + namespace: Dir, objects: Dir, manifests: Dir, @@ -90,9 +92,21 @@ impl cyber_server::runtime::SnapshotNativeGuard for WorkerGuard { } impl Store { pub fn open(data: &Path, location: &str, options: Options) -> Result { - Self::open_mode(data, location, options, true) + Self::open_mode(data, location, options, true, true) } pub fn existing(data: &Path, location: &str, options: Options) -> Result> { + Self::existing_mode(data, location, options, false) + } + pub fn recover_existing(data: &Path, location: &str, options: Options) -> Result { + Self::existing_mode(data, location, options, true)? + .ok_or("No retained file-copy namespace for this Location".into()) + } + fn existing_mode( + data: &Path, + location: &str, + options: Options, + own: bool, + ) -> Result> { if !data.exists() { return Ok(None); } @@ -112,9 +126,15 @@ impl Store { if global.symlink_metadata(&key).is_err() { return Ok(None); } - Self::open_mode(data, location, options, false).map(Some) + Self::open_mode(data, location, options, false, own).map(Some) } - fn open_mode(data: &Path, location: &str, mut options: Options, create: bool) -> Result { + fn open_mode( + data: &Path, + location: &str, + mut options: Options, + create: bool, + own: bool, + ) -> Result { options.private_roots = options .private_roots .into_iter() @@ -160,8 +180,8 @@ impl Store { let copy = child(&snapshot, "global", true)?; let key = storage::digest(format!("{}\0{}", root.display(), root_id).as_bytes()); let dir = child(©, &key, true)?; - let lock = if create { - Some(storage::lock(&dir, true)?) + let lock = if own { + Some(storage::lock(&dir, create)?) } else { None }; @@ -173,6 +193,7 @@ impl Store { root_id, data_path, work, + namespace: dir, objects, manifests, journals, @@ -196,6 +217,26 @@ impl Store { } Ok(()) } + pub(super) fn lock_identity(&self) -> Result { + storage::identity( + self._lock + .as_ref() + .ok_or("Snapshot mutation lock is not held")?, + ) + } + pub(super) fn verify_lock_route(&self, expected: &str) -> Result<()> { + let current = storage::open(&self.namespace, Path::new("lock"))?; + storage::validate_lock(¤t)?; + if storage::identity(¤t)? != expected + || self + ._lock + .as_ref() + .is_some_and(|held| storage::identity(held).as_deref() != Ok(expected)) + { + return Err("Snapshot private lock identity changed".into()); + } + Ok(()) + } fn usage(&self) -> Result { if let Some(size) = self.usage.get() { return Ok(size); diff --git a/crates/cyber-snapshot/src/copy/ownership.rs b/crates/cyber-snapshot/src/copy/ownership.rs index 0346c9874..9b5af03eb 100644 --- a/crates/cyber-snapshot/src/copy/ownership.rs +++ b/crates/cyber-snapshot/src/copy/ownership.rs @@ -8,6 +8,157 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; use std::path::{Component, Path}; +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ImageObservation { + pub identity: String, + pub entry: Entry, + pub generation: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Slots { + pub source: Option, + pub stage: Option, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Observations { + pub files: Vec, + parents: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ParentObservation { + path: String, + source: Option, + stage: Option, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DirectoryObservation { + identity: String, + generation: String, +} + +impl Observations { + pub fn validate(&self) -> Result<()> { + if self.files.len() > ENTRY_LIMIT || self.parents.len() > ENTRY_LIMIT + 1 { + return Err("Recovery observation limit exceeded".into()); + } + for slots in &self.files { + for image in [&slots.source, &slots.stage].into_iter().flatten() { + if !valid_identity(&image.identity) + || !valid_entry(&image.entry) + || !storage::valid_digest(&image.generation) + { + return Err("Invalid recovery file observation".into()); + } + } + } + for parent in &self.parents { + if !parent.path.is_empty() { + super::tree::safe_path(&parent.path)?; + } + for directory in [&parent.source, &parent.stage].into_iter().flatten() { + if !valid_identity(&directory.identity) + || !storage::valid_digest(&directory.generation) + { + return Err("Invalid recovery parent observation".into()); + } + } + } + Ok(()) + } + pub fn budget(&self) -> Self { + let mut value = self.clone(); + let image = ImageObservation { + identity: "0".repeat(storage::IDENTITY_MAX_BYTES), + generation: "0".repeat(64), + entry: Entry { + object: "0".repeat(64), + length: u64::MAX, + mode: u32::MAX, + uid: u32::MAX, + gid: u32::MAX, + }, + }; + for slots in &mut value.files { + slots.source = Some(image.clone()); + slots.stage = Some(image.clone()); + } + for parent in &mut value.parents { + let bound = DirectoryObservation { + identity: "0".repeat(storage::IDENTITY_MAX_BYTES), + generation: "0".repeat(64), + }; + parent.source = Some(bound.clone()); + parent.stage = Some(bound); + } + value + } +} + +fn generation(file: &std::fs::File) -> Result { + use std::os::unix::fs::MetadataExt; + let m = file.metadata().map_err(fail)?; + let value = ( + m.dev(), + m.ino(), + m.mode(), + m.uid(), + m.gid(), + m.nlink(), + m.len(), + m.mtime(), + m.mtime_nsec(), + m.ctime(), + m.ctime_nsec(), + ); + Ok(storage::digest(&serde_json::to_vec(&value).map_err(fail)?)) +} + +pub(super) fn observe_image( + store: &Store, + parent: Option<&Dir>, + name: &std::ffi::OsStr, +) -> Result> { + let Some(parent) = parent else { + return Ok(None); + }; + match parent.symlink_metadata(name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(fail(error)), + Ok(_) => {} + } + let pin = storage::open(parent, Path::new(name))?; + let before = generation(&pin)?; + let (identity, entry) = + observed(store, Some(parent), name)?.ok_or("Recovery image disappeared")?; + let current = storage::open(parent, Path::new(name))?; + if storage::identity(&pin)? != identity + || storage::identity(¤t)? != identity + || generation(&pin)? != before + || generation(¤t)? != before + { + return Err("Recovery image changed while reviewing".into()); + } + Ok(Some(ImageObservation { + identity, + entry, + generation: before, + })) +} + +fn observe_directory(dir: &Dir) -> Result { + let file = dir.try_clone().map_err(fail)?.into_std_file(); + Ok(DirectoryObservation { + identity: storage::identity(&file)?, + generation: generation(&file)?, + }) +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct Ownership { @@ -211,6 +362,101 @@ impl File { } impl Ownership { + pub(super) fn created_parents(&self) -> Vec { + self.parents + .iter() + .filter(|parent| parent.stage.is_some()) + .map(|parent| parent.path.clone()) + .collect() + } + /// Capture ordinary current files without assigning foreign files to this restore. + /// Keep can preserve these images; rollback still requires strict `inspect` proof. + pub(super) fn observe( + &self, + store: &Store, + journal: &Dir, + changes: &[Change], + ) -> Result { + self.validate(changes)?; + self.observe_current(store, journal, changes) + } + pub(super) fn observe_unprepared( + store: &Store, + journal: &Dir, + changes: &[Change], + ) -> Result { + Self::budget(changes)?.observe_current(store, journal, changes) + } + fn observe_current( + &self, + store: &Store, + journal: &Dir, + changes: &[Change], + ) -> Result { + store.verify_root()?; + let mut parents = vec![ParentObservation { + path: String::new(), + source: Some(observe_directory(&store.work)?), + stage: None, + }]; + for parent in &self.parents { + let live = existing_parent(store, &parent.path)? + .as_ref() + .map(observe_directory) + .transpose()?; + let stage = parent + .stage + .as_ref() + .map(|stage| match journal.symlink_metadata(stage) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(fail(error)), + Ok(_) => observe_directory(&journal.open_dir_nofollow(stage).map_err(fail)?) + .map(Some), + }) + .transpose()? + .flatten(); + parents.push(ParentObservation { + path: parent.path.clone(), + source: live, + stage, + }); + } + let mut files = vec![]; + for (index, change) in changes.iter().enumerate() { + store.check_source_path(&change.path)?; + let parent_path = Path::new(&change.path) + .parent() + .ok_or("Invalid recovery parent")?; + let parent = existing_parent( + store, + parent_path.to_str().ok_or("Invalid recovery parent")?, + )?; + let leaf = Path::new(&change.path) + .file_name() + .ok_or("Invalid recovery path")?; + files.push(Slots { + source: observe_image(store, parent.as_ref(), leaf)?, + stage: observe_image( + store, + Some(journal), + std::ffi::OsStr::new(&format!("stage_{index}")), + )?, + }); + } + // Detect route changes during the bounded scan, including parent edits + // that do not replace any of the tracked leaf files. + for parent in &parents { + let now = existing_parent(store, &parent.path)? + .as_ref() + .map(observe_directory) + .transpose()?; + if now != parent.source { + return Err("Recovery parent changed while reviewing".into()); + } + } + store.verify_root()?; + Ok(Observations { files, parents }) + } pub(super) fn verify_original(&self, index: usize, file: &std::fs::File) -> Result<()> { if self.files.get(index).and_then(|file| file.source.as_ref()) != Some(&storage::identity(file)?) diff --git a/crates/cyber-snapshot/src/copy/ownership_tests.rs b/crates/cyber-snapshot/src/copy/ownership_tests.rs index 35e64d055..d13fe565b 100644 --- a/crates/cyber-snapshot/src/copy/ownership_tests.rs +++ b/crates/cyber-snapshot/src/copy/ownership_tests.rs @@ -1,4 +1,6 @@ //! Real file/parent namespace operations; no runtime acknowledgement is asserted. +#[path = "resolution_tests.rs"] +mod resolution_tests; use super::*; use cap_std::fs::Dir; use std::path::PathBuf; @@ -51,7 +53,7 @@ impl Prepared { }, ]; let mut record = store - .prepare_journal(changes, "restore_fixture", None) + .prepare_journal(changes, "restore_fixture", None, None, None) .unwrap(); let dir = storage::private_child(&store.journals, "restore_fixture").unwrap(); Store::record(&dir, &record).unwrap(); diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs new file mode 100644 index 000000000..75c78865c --- /dev/null +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -0,0 +1,518 @@ +//! A resolution is an independent intent. Filesystem success is not native acknowledgement. +use super::Store; +use super::ownership::{Effect, ImageObservation, Observations, Ownership, observe_image}; +use super::restore::{Change, Journal}; +use super::storage::{self, Result, fail}; +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; +use cyber_server::runtime::{ + RestoreError, SnapshotBinding, SnapshotOrigin, SnapshotRecoveryAction as Action, + SnapshotRecoveryConfirmation as Confirmation, SnapshotRecoveryImage as Image, + SnapshotRecoveryPath as PathView, SnapshotRecoveryReceipt as Receipt, + SnapshotRecoveryReview as Review, SnapshotWork, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Resolution { + pub request: Confirmation, + pub original: SnapshotOrigin, + pub original_digest: String, + pub review: Observations, + pub outcome: Option, +} + +impl Resolution { + pub fn validate(&self, id: &str, record: &Journal) -> Result<()> { + validate_journal_id(&self.request.journal_id)?; + if self.original.version != 1 + || id == self.request.journal_id + || self.original.binding.journal_id != self.request.journal_id + || resolution_id(&self.original, &self.request.request_id)? != id + || !storage::valid_digest(&self.original_digest) + || !storage::valid_digest(&self.request.confirmation) + || self.outcome.is_some() && !record.completed + { + return Err("Invalid recovery resolution intent".into()); + } + if let Some(origin) = &record.origin + && (origin.session_id != self.original.session_id + || origin.session_created_id != self.original.session_created_id + || origin.worktree_id != self.original.worktree_id + || origin.activity_id == self.original.activity_id) + { + return Err("Resolution origin differs from original Session/checkout".into()); + } + self.review.validate()?; + if let Some(outcome) = &self.outcome { + outcome.validate()?; + } + Ok(()) + } +} + +struct Plan { + review: Review, + observations: Observations, + changes: Vec, + digest: String, +} + +pub(super) struct Preimages<'a> { + dir: Dir, + parent: Journal, + paths: std::collections::BTreeMap, + review: &'a Observations, +} +impl Preimages<'_> { + pub fn verify_review(&self, store: &Store) -> Result<()> { + if &observe(store, &self.dir, &self.parent)? != self.review { + return Err("Recovery owner state changed during private preparation".into()); + } + Ok(()) + } + pub fn check_source(&self, store: &Store, path: &str) -> Result<()> { + let index = *self.paths.get(path).ok_or("Unreviewed recovery source")?; + let parent = self + .parent + .ownership + .as_ref() + .ok_or("Missing original ownership")? + .parent(store, &self.dir, path)?; + let leaf = std::path::Path::new(path) + .file_name() + .ok_or("Invalid recovery source")?; + if observe_image(store, parent.as_ref(), leaf)? != self.review.files[index].source { + return Err("Recovery source differs from its reviewed native image".into()); + } + Ok(()) + } + pub fn bytes(&self, store: &Store, path: &str, expected: &super::Entry) -> Result> { + let index = *self + .paths + .get(path) + .ok_or("Resolution path is absent from original intent")?; + if self.parent.changes[index].before.as_ref() != Some(expected) { + return Err("Resolution image differs from original preimage".into()); + } + let observed = self + .review + .files + .get(index) + .and_then(|slots| slots.stage.as_ref()) + .ok_or("Original native preimage is unavailable")?; + let name = format!("stage_{index}"); + if observe_image(store, Some(&self.dir), std::ffi::OsStr::new(&name))?.as_ref() + != Some(observed) + { + return Err("Original native preimage changed before preparation".into()); + } + let bytes = storage::read(&self.dir, &name, expected.length)?; + if bytes.len() as u64 != expected.length + || storage::digest(&bytes) != expected.object + || observe_image(store, Some(&self.dir), std::ffi::OsStr::new(&name))?.as_ref() + != Some(observed) + { + return Err("Original native preimage changed while preparing resolution".into()); + } + Ok(bytes) + } +} + +fn image(value: &ImageObservation) -> Image { + expected_image(&value.entry, Some(value.identity.clone())) +} +fn expected_image(entry: &super::Entry, native_identity: Option) -> Image { + Image { + hash: entry.object.clone(), + bytes: entry.length, + mode: entry.mode, + uid: entry.uid, + gid: entry.gid, + native_identity, + } +} +fn valid_id(id: &str) -> bool { + !id.is_empty() && id.len() <= 256 && !id.contains('\0') +} +fn resolution_id(original: &SnapshotOrigin, request: &str) -> Result { + if !valid_id(request) { + return Err("Invalid snapshot recovery request ID".into()); + } + let key = serde_json::to_vec(&(original.session_created_id.as_str(), request)).map_err(fail)?; + Ok(format!("restore_resolution_{}", storage::digest(&key))) +} +fn immutable_digest(record: &Journal) -> Result { + Ok(storage::digest( + &serde_json::to_vec(&( + record.version, + &record.location, + &record.root, + record.created, + &record.changes, + &record.origin, + &record.ownership, + &record.context, + &record.resolution, + )) + .map_err(fail)?, + )) +} + +impl Store { + pub(super) fn resolution_preimages<'a>( + &self, + resolution: Option<&'a Resolution>, + ) -> Result>> { + let Some(resolution) = resolution.filter(|value| value.request.action == Action::Rollback) + else { + return Ok(None); + }; + self.verify_worker_namespace(&resolution.original)?; + let (dir, parent) = self.journal(&resolution.original.binding.journal_id)?; + if immutable_digest(&parent)? != resolution.original_digest { + return Err("Original intent changed before preimage preparation".into()); + } + let paths = parent + .changes + .iter() + .enumerate() + .map(|(index, change)| (change.path.clone(), index)) + .collect(); + Ok(Some(Preimages { + dir, + parent, + paths, + review: &resolution.review, + })) + } + pub fn recovery_review(&self, id: &str, action: Action) -> Result { + Ok(self.resolution_plan(id, action)?.review) + } + + fn resolution_plan(&self, id: &str, action: Action) -> Result { + validate_journal_id(id)?; + self.verify_root()?; + let (dir, record) = self.journal(id)?; + let observations = observe(self, &dir, &record)?; + let digest = immutable_digest(&record)?; + let mut review = Review { + journal_id: id.into(), + action, + origin: record.origin.clone(), + filesystem_complete: record.completed, + acknowledged: record.acknowledged, + confirmation: None, + paths: vec![], + exclusions: record + .context + .as_ref() + .map(|context| context.exclusions.clone()), + recorded_created_parents: record.ownership.as_ref().map(Ownership::created_parents), + conflicts: vec![], + }; + if record.acknowledged { + review + .conflicts + .push("This restore has already been acknowledged".into()); + } + if record.origin.is_none() { + review + .conflicts + .push("Legacy intent has no exact native worker origin".into()); + } + if let Some(origin) = &record.origin + && let Err(error) = self.verify_worker_namespace(origin) + { + review.conflicts.push(error); + } + if action == Action::Rollback { + match &record.ownership { + None => review + .conflicts + .push("Rollback requires prepared native file ownership".into()), + Some(proof) => { + if let Err(error) = proof.inspect(self, &dir, &record.changes) { + review.conflicts.push(error); + } + } + } + } + let mut changes = vec![]; + for (index, (change, slots)) in record.changes.iter().zip(&observations.files).enumerate() { + let effect = record + .ownership + .as_ref() + .and_then(|proof| proof.inspect_change(self, &dir, index, change).ok()); + let reverse = action == Action::Rollback && effect == Some(Effect::Applied); + let current = slots.source.as_ref().map(image); + let proposed = if reverse { + change + .before + .as_ref() + .map(|entry| expected_image(entry, None)) + } else { + current.clone() + }; + review.paths.push(PathView { + path: change.path.clone(), + state: match effect { + Some(Effect::Applied) => "applied", + Some(Effect::Unapplied) => "unapplied", + None => "foreign", + } + .into(), + current, + recorded_before: change + .before + .as_ref() + .map(|entry| expected_image(entry, None)), + recorded_after: change + .after + .as_ref() + .map(|entry| expected_image(entry, None)), + retained: slots.stage.as_ref().map(image), + proposed, + changes: reverse, + }); + if reverse { + changes.push(Change { + path: change.path.clone(), + before: slots.source.as_ref().map(|value| value.entry.clone()), + after: change.before.clone(), + }); + } + } + let binding = SnapshotBinding { + directory: self.root.clone(), + root_identity: self.root_id.clone(), + namespace_identity: storage::dir_identity(&self.journals)?, + journal_id: id.into(), + journal_identity: storage::dir_identity(&dir)?, + lock_identity: record + .origin + .as_ref() + .and_then(|origin| origin.binding.lock_identity.clone()), + }; + if review.conflicts.is_empty() { + let raw = serde_json::to_vec(&( + 1u32, + action, + &binding, + &digest, + &observations, + record.completed, + record.acknowledged, + )) + .map_err(fail)?; + review.confirmation = Some(storage::digest(&raw)); + } + if serde_json::to_vec(&review).map_err(fail)?.len() > 16 * 1024 * 1024 { + return Err("Recovery review exceeds response limit".into()); + } + // Re-read control data; a live writer may have advanced preparation or progress. + let (_, current) = self.journal(id)?; + if immutable_digest(¤t)? != digest + || current.completed != record.completed + || current.applied != record.applied + || current.acknowledged != record.acknowledged + { + return Err("Recovery intent changed while reviewing".into()); + } + Ok(Plan { + review, + observations, + changes, + digest, + }) + } + + /// Native backend only. Consuming runtime acknowledgement is a separate owner transaction. + pub fn resolve_recovery( + &self, + request: &Confirmation, + authorize: &(dyn Fn() -> Result<()> + Send + Sync), + worker: Option<&mut SnapshotWork<'_>>, + ) -> std::result::Result { + validate_journal_id(&request.journal_id).map_err(RestoreError::Failed)?; + let (parent_dir, parent) = self + .journal(&request.journal_id) + .map_err(RestoreError::Failed)?; + let original = parent + .origin + .clone() + .ok_or_else(|| RestoreError::Failed("Recovery requires exact native origin".into()))?; + let id = resolution_id(&original, &request.request_id).map_err(RestoreError::Failed)?; + // A durable prior request is lookup-only, including failed/partial outcomes. + if self.journals.symlink_metadata(&id).is_ok() { + return self + .lookup_resolution(&id, request, &original) + .map_err(RestoreError::Failed); + } + authorize().map_err(RestoreError::Failed)?; + self.verify_worker_namespace(&original) + .map_err(RestoreError::Failed)?; + let plan = self + .resolution_plan(&request.journal_id, request.action) + .map_err(RestoreError::Failed)?; + if plan.review.confirmation.as_deref() != Some(&request.confirmation) { + return Err(RestoreError::Stale); + } + let resolution = Resolution { + request: request.clone(), + original: original.clone(), + original_digest: plan.digest, + review: plan.observations.clone(), + outcome: None, + }; + let mut record = self + .prepare_journal( + plan.changes, + &id, + worker.as_deref(), + parent.context.clone(), + Some(resolution), + ) + .map_err(RestoreError::Failed)?; + if let Some(worker) = worker.as_deref() { + let source = worker.preview_origin(original.binding.clone()); + if source.session_id != original.session_id + || source.session_created_id != original.session_created_id + || source.worktree_id != original.worktree_id + { + return Err(RestoreError::Failed( + "Recovery worker differs from the original Session/checkout".into(), + )); + } + } + // Revalidate after storage waits/reservation, before publishing the distinct intent. + let fresh = self + .resolution_plan(&request.journal_id, request.action) + .map_err(RestoreError::Failed)?; + if fresh.review.confirmation.as_deref() != Some(&request.confirmation) { + return Err(RestoreError::Stale); + } + self.execute_journal(&id, &mut record, authorize, worker)?; + let dir = self + .journals + .open_dir_nofollow(&id) + .map_err(|e| RestoreError::Failed(fail(e)))?; + let outcome = observe(self, &parent_dir, &parent).map_err(RestoreError::Failed)?; + verify_unchanged_slots( + &plan.observations, + &outcome, + &parent.changes, + &record.changes, + ) + .map_err(RestoreError::Failed)?; + record + .resolution + .as_mut() + .ok_or_else(|| RestoreError::Failed("Missing resolution intent".into()))? + .outcome = Some(outcome); + Self::record(&dir, &record).map_err(RestoreError::Failed)?; + self.verify_resolution(&dir, &record) + .map_err(RestoreError::Failed)?; + receipt(&id, &record).map_err(RestoreError::Failed) + } + + fn lookup_resolution( + &self, + id: &str, + request: &Confirmation, + original: &SnapshotOrigin, + ) -> Result { + let (_, record) = self.journal(id)?; + let resolution = record + .resolution + .as_ref() + .ok_or("Recovery request is not a resolution")?; + if &resolution.request != request || &resolution.original != original { + return Err( + "Recovery request ID was reused with different ownership or content".into(), + ); + } + receipt(id, &record) + } + + pub(super) fn verify_resolution(&self, _dir: &Dir, record: &Journal) -> Result<()> { + let Some(resolution) = &record.resolution else { + return Ok(()); + }; + self.verify_worker_namespace(&resolution.original)?; + let (parent_dir, parent) = self.journal(&resolution.original.binding.journal_id)?; + if immutable_digest(&parent)? != resolution.original_digest { + return Err("Original recovery intent changed".into()); + } + let expected = resolution + .outcome + .as_ref() + .ok_or("Recovery filesystem outcome is incomplete; lookup without replay")?; + if &observe(self, &parent_dir, &parent)? != expected { + return Err("Recovery outcome changed before acknowledgement".into()); + } + Ok(()) + } +} + +fn validate_journal_id(id: &str) -> Result<()> { + if !id.starts_with("restore_") + || id.len() > 256 + || !id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') + { + return Err("Invalid snapshot recovery journal ID".into()); + } + Ok(()) +} +fn observe(store: &Store, dir: &Dir, record: &Journal) -> Result { + match &record.ownership { + Some(proof) => proof.observe(store, dir, &record.changes), + None => Ownership::observe_unprepared(store, dir, &record.changes), + } +} +fn verify_unchanged_slots( + before: &Observations, + after: &Observations, + original: &[Change], + changes: &[Change], +) -> Result<()> { + if before.files.len() != after.files.len() { + return Err("Recovery inventory changed".into()); + } + // Every original private slot is retained. Source changes are independently + // proven by the new resolution's native ownership before its final commit. + for ((old, new), original) in before.files.iter().zip(&after.files).zip(original) { + if old.stage != new.stage { + return Err("Original recovery preimages changed".into()); + } + if !changes.iter().any(|change| change.path == original.path) && old.source != new.source { + return Err("Unchanged owner file changed during recovery".into()); + } + } + if changes.is_empty() && before != after { + return Err("Owner state changed during keep".into()); + } + Ok(()) +} +fn receipt(id: &str, record: &Journal) -> Result { + let resolution = record + .resolution + .as_ref() + .ok_or("Missing resolution record")?; + if !record.completed || resolution.outcome.is_none() { + return Err( + "Recovery request has an incomplete outcome; inspect retained intent, do not replay" + .into(), + ); + } + Ok(Receipt { + journal_id: resolution.request.journal_id.clone(), + resolution_id: id.into(), + request_id: resolution.request.request_id.clone(), + action: resolution.request.action, + confirmation: resolution.request.confirmation.clone(), + original: resolution.original.clone(), + filesystem_complete: true, + }) +} diff --git a/crates/cyber-snapshot/src/copy/resolution_tests.rs b/crates/cyber-snapshot/src/copy/resolution_tests.rs new file mode 100644 index 000000000..7ff65a377 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/resolution_tests.rs @@ -0,0 +1,468 @@ +//! Native backend fixtures. Synthetic origins confer no runtime acknowledgement authority. +use super::*; +use cyber_server::runtime::{ + SnapshotRecoveryAction as Action, SnapshotRecoveryConfirmation as Confirmation, +}; + +impl Prepared { + fn managed() -> Self { + let mut p = Self::new(); + p.record.origin = Some(cyber_server::runtime::SnapshotOrigin { + version: 1, + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + activity_id: "op_fixture".into(), + worktree_id: None, + binding: cyber_server::runtime::SnapshotBinding { + directory: p.store.root.clone(), + root_identity: p.store.root_id.clone(), + namespace_identity: storage::dir_identity(&p.store.journals).unwrap(), + journal_id: "restore_fixture".into(), + journal_identity: storage::dir_identity(&p.dir).unwrap(), + lock_identity: Some(p.store.lock_identity().unwrap()), + }, + }); + Store::record(&p.dir, &p.record).unwrap(); + p + } + fn confirmation(&self, action: Action, id: &str) -> Confirmation { + let review = self + .store + .recovery_review("restore_fixture", action) + .unwrap(); + assert!(review.conflicts.is_empty(), "{:?}", review.conflicts); + Confirmation { + journal_id: "restore_fixture".into(), + action, + confirmation: review.confirmation.unwrap(), + request_id: id.into(), + } + } + fn resolve(&self, request: &Confirmation) -> cyber_server::runtime::SnapshotRecoveryReceipt { + self.store + .resolve_recovery(request, &|| Ok(()), None) + .unwrap() + } +} + +#[test] +fn rollback_reverses_proven_files_and_retains_actual_displaced_live_inodes() { + let p = Prepared::managed(); + for index in 0..3 { + p.apply(index); + } + std::fs::write(p.root.join("owner"), b"keep owner data").unwrap(); + let live = storage::identity( + &storage::open( + &p.store.parent("existing/replace").unwrap(), + Path::new("replace"), + ) + .unwrap(), + ) + .unwrap(); + let created = storage::identity( + &storage::open(&p.store.parent("new/deep/file").unwrap(), Path::new("file")).unwrap(), + ) + .unwrap(); + let request = p.confirmation(Action::Rollback, "rollback-one"); + let receipt = p.resolve(&request); + assert!(receipt.filesystem_complete); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"before" + ); + assert_eq!( + std::fs::read(p.root.join("existing/delete")).unwrap(), + b"deleted" + ); + assert!(!p.root.join("new/deep/file").exists()); + assert_eq!( + std::fs::read(p.root.join("owner")).unwrap(), + b"keep owner data" + ); + let (dir, record) = p.store.journal(&receipt.resolution_id).unwrap(); + assert_eq!( + storage::identity(&storage::open(&dir, Path::new("stage_0")).unwrap()).unwrap(), + live + ); + assert_eq!( + storage::identity(&storage::open(&dir, Path::new("stage_2")).unwrap()).unwrap(), + created + ); + assert!(record.completed && !record.acknowledged); + assert!(!p.store.journal("restore_fixture").unwrap().1.acknowledged); + assert!(p.store.ensure_recovered().is_err()); + assert_eq!(storage::read(&p.dir, "stage_0", 65536).unwrap(), b"before"); + assert_eq!(storage::read(&p.dir, "stage_1", 65536).unwrap(), b"deleted"); +} + +#[test] +fn keep_preserves_reviewed_foreign_owner_images_without_assigning_restore_ownership() { + let p = Prepared::managed(); + p.apply(0); + p.replace_same_bytes(&p.root.join("existing/replace"), b"my later edit"); + let rollback = p + .store + .recovery_review("restore_fixture", Action::Rollback) + .unwrap(); + assert!(rollback.confirmation.is_none() && !rollback.conflicts.is_empty()); + let keep = p + .store + .recovery_review("restore_fixture", Action::Keep) + .unwrap(); + assert_eq!(keep.paths[0].state, "foreign"); + assert!(keep.paths.iter().all(|path| !path.changes)); + let request = p.confirmation(Action::Keep, "keep-one"); + let receipt = p.resolve(&request); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"my later edit" + ); + let (dir, record) = p.store.journal(&receipt.resolution_id).unwrap(); + assert!(record.changes.is_empty() && !record.acknowledged); + assert!(p.store.verify_resolution(&dir, &record).is_ok()); + std::fs::write(p.root.join("existing/replace"), b"changed after completion").unwrap(); + assert!(p.store.verify_resolution(&dir, &record).is_err()); + let retry = p + .store + .resolve_recovery( + &request, + &|| panic!("retry must not authorize effects"), + None, + ) + .unwrap(); + assert_eq!(retry, receipt); + assert!(p.store.ensure_recovered().is_err()); +} + +#[test] +fn stale_generation_and_whole_plan_conflicts_refuse_before_intent_or_effect() { + let p = Prepared::managed(); + for index in 0..3 { + p.apply(index); + } + let request = p.confirmation(Action::Rollback, "stale-one"); + std::fs::File::open(p.root.join("existing/replace")) + .unwrap() + .set_modified(std::time::SystemTime::now() + std::time::Duration::from_secs(17)) + .unwrap(); + assert!(matches!( + p.store.resolve_recovery(&request, &|| Ok(()), None), + Err(RestoreError::Stale) + )); + let request = p.confirmation(Action::Rollback, "stale-two"); + std::fs::write(p.root.join("existing/delete"), b"new owner file").unwrap(); + assert!( + p.store + .resolve_recovery(&request, &|| Ok(()), None) + .is_err() + ); + assert_eq!( + storage::names(&p.store.journals, 100).unwrap(), + ["restore_fixture"] + ); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"after" + ); + assert_eq!( + std::fs::read(p.root.join("existing/delete")).unwrap(), + b"new owner file" + ); +} + +#[test] +fn a_same_inode_generation_change_before_the_effect_is_not_adopted_as_new_ownership() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "generation-during-prepare"); + let calls = AtomicUsize::new(0); + let authorize = || { + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + std::fs::File::open(p.root.join("existing/replace")) + .unwrap() + .set_modified(std::time::SystemTime::now() + std::time::Duration::from_secs(31)) + .unwrap(); + } + Ok(()) + }; + assert!( + p.store + .resolve_recovery(&request, &authorize, None) + .is_err() + ); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"after" + ); + let ids = storage::names(&p.store.journals, 100).unwrap(); + assert_eq!(ids.len(), 2); + let (_, record) = p + .store + .journal( + ids.iter() + .find(|id| id.as_str() != "restore_fixture") + .unwrap(), + ) + .unwrap(); + assert!(record.applied.is_empty() && !record.completed); + assert!(p.store.ensure_recovered().is_err()); +} + +#[test] +fn interrupted_resolution_is_independent_evidence_and_retry_never_redispatches() { + let p = Prepared::managed(); + for index in 0..3 { + p.apply(index); + } + let request = p.confirmation(Action::Rollback, "partial-one"); + let first = p.root.join("existing/replace"); + let authorize = || { + if std::fs::read(&first).unwrap() == b"before" { + Err("admission revoked".into()) + } else { + Ok(()) + } + }; + assert!( + p.store + .resolve_recovery(&request, &authorize, None) + .is_err() + ); + let ids = storage::names(&p.store.journals, 100).unwrap(); + assert_eq!(ids.len(), 2); + let id = ids + .iter() + .find(|id| id.as_str() != "restore_fixture") + .unwrap(); + let (_, record) = p.store.journal(id).unwrap(); + assert!(!record.completed); + assert_eq!(record.applied, ["existing/replace"]); + assert!(record.resolution.as_ref().unwrap().outcome.is_none()); + assert!( + p.store + .resolve_recovery( + &request, + &|| panic!("partial retry must not dispatch"), + None + ) + .is_err() + ); + assert_eq!(std::fs::read(&first).unwrap(), b"before"); + assert!(!p.root.join("existing/delete").exists()); + assert_eq!( + std::fs::read(p.root.join("new/deep/file")).unwrap(), + b"after" + ); + assert!(p.store.ensure_recovered().is_err()); +} + +#[test] +fn same_request_id_with_different_action_or_fingerprint_refuses_without_replay() { + let p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "owned-id"); + let receipt = p.resolve(&request); + for action in [Action::Keep, Action::Rollback] { + let mut changed = request.clone(); + changed.action = action; + changed.confirmation = "0".repeat(64); + assert!( + p.store + .resolve_recovery(&changed, &|| panic!("ID conflict must not dispatch"), None) + .is_err() + ); + } + assert_eq!(p.resolve(&request), receipt); + assert_eq!(storage::names(&p.store.journals, 100).unwrap().len(), 2); +} + +#[test] +fn cap_refusal_preserves_both_review_and_preimages_until_capacity_is_available() { + let mut p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "cap-one"); + p.store.options.max_total = 1; + assert!( + p.store + .resolve_recovery(&request, &|| Ok(()), None) + .is_err() + ); + assert_eq!( + storage::names(&p.store.journals, 100).unwrap(), + ["restore_fixture"] + ); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"after" + ); + assert_eq!(storage::read(&p.dir, "stage_0", 65536).unwrap(), b"before"); + p.store.options.max_total = 4 * 1024 * 1024; + assert!(p.resolve(&request).filesystem_complete); +} + +#[test] +fn rollback_reads_the_actual_retained_preimage_independently_of_capture_objects() { + let p = Prepared::managed(); + p.apply(0); + let hash = &p.record.changes[0].before.as_ref().unwrap().object; + p.store.objects.remove_file(hash).unwrap(); + let receipt = p.resolve(&p.confirmation(Action::Rollback, "retained-preimage")); + assert!(receipt.filesystem_complete); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"before" + ); + assert_eq!(storage::read(&p.dir, "stage_0", 65536).unwrap(), b"before"); + assert!(p.store.objects.symlink_metadata(hash).is_err()); +} + +#[test] +fn legacy_and_unprepared_journals_do_not_acquire_rollback_proof() { + let mut p = Prepared::managed(); + p.record.ownership = None; + Store::record(&p.dir, &p.record).unwrap(); + assert!( + p.store + .recovery_review("restore_fixture", Action::Rollback) + .unwrap() + .confirmation + .is_none() + ); + assert!( + p.resolve(&p.confirmation(Action::Keep, "unprepared")) + .filesystem_complete + ); + p.record.origin = None; + p.record.version = 1; + Store::record(&p.dir, &p.record).unwrap(); + let review = p + .store + .recovery_review("restore_fixture", Action::Keep) + .unwrap(); + assert!(review.confirmation.is_none()); + assert!(review.exclusions.is_none()); +} + +#[tokio::test] +async fn host_review_ignores_capture_disable_and_lowered_capture_size_without_writing() { + use cyber_server::runtime::Snapshots; + let p = Prepared::managed(); + p.apply(0); + let before = storage::names(&p.store.objects, 100).unwrap(); + for config in [ + serde_json::json!({"snapshots":false}), + serde_json::json!({"snapshots":{"max_file_bytes":1,"fallback_max_bytes":1}}), + ] { + let snapshots = crate::GitSnapshots::new( + p.store.data_path.clone(), + std::sync::Arc::new(move |_| config.clone()), + ); + let view = snapshots + .recovery_review(&p.root.to_string_lossy(), "restore_fixture", Action::Keep) + .await + .unwrap(); + assert!(view.confirmation.is_some()); + assert_eq!(view.paths[0].current.as_ref().unwrap().bytes, 5); + } + assert_eq!(storage::names(&p.store.objects, 100).unwrap(), before); + assert_eq!( + storage::names(&p.store.journals, 100).unwrap(), + ["restore_fixture"] + ); +} + +#[test] +fn replaced_lock_file_cannot_prove_a_live_original_worker_has_exited() { + let p = Prepared::managed(); + let request = p.confirmation(Action::Keep, "lock-bound"); + let namespace = p + .private_path("intent.json") + .parent() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .to_owned(); + let lock = namespace.join("lock"); + std::fs::rename(&lock, namespace.join("old_lock")).unwrap(); + std::fs::write(&lock, b"").unwrap(); + std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o600)).unwrap(); + // A different lock inode really can be acquired while the old owner is live. + let foreign = Store::recover_existing( + &p.store.data_path, + &p.root.to_string_lossy(), + p.store.options.clone(), + ) + .unwrap(); + assert_ne!( + foreign.lock_identity().unwrap(), + p.store.lock_identity().unwrap() + ); + let review = foreign + .recovery_review("restore_fixture", Action::Keep) + .unwrap(); + assert!(review.confirmation.is_none()); + assert!( + review + .conflicts + .iter() + .any(|message| message.contains("lock identity changed")) + ); + assert!( + foreign + .resolve_recovery(&request, &|| Ok(()), None) + .is_err() + ); + assert!( + p.store + .verify_worker_namespace(p.record.origin.as_ref().unwrap()) + .is_err() + ); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"before" + ); +} + +#[test] +fn legacy_origin_without_exact_lock_binding_remains_inspectable_and_fenced() { + let mut p = Prepared::managed(); + p.record.origin.as_mut().unwrap().binding.lock_identity = None; + Store::record(&p.dir, &p.record).unwrap(); + let view = p + .store + .recovery_review("restore_fixture", Action::Keep) + .unwrap(); + assert!(view.confirmation.is_none()); + assert!( + view.conflicts + .iter() + .any(|message| message.contains("no exact lock identity")) + ); + assert!(p.store.ensure_recovered().is_err()); +} + +#[test] +fn recovery_requires_an_existing_namespace_and_real_exclusive_lock() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work"); + std::fs::create_dir(&root).unwrap(); + let data = temp.path().join("data"); + let options = super::super::super::Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }; + assert!(Store::recover_existing(&data, &root.to_string_lossy(), options.clone()).is_err()); + assert!(!data.exists()); + let first = Store::open(&data, &root.to_string_lossy(), options.clone()).unwrap(); + assert!(Store::recover_existing(&data, &root.to_string_lossy(), options.clone()).is_err()); + let identity = first.lock_identity().unwrap(); + drop(first); + let second = Store::recover_existing(&data, &root.to_string_lossy(), options).unwrap(); + assert_eq!(second.lock_identity().unwrap(), identity); +} diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 1f315813b..7b70140d6 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -1,5 +1,6 @@ //! A durable intent precedes every namespace mutation; incomplete intents fence reuse. use super::ownership::{Effect, Ownership}; +use super::resolution::Resolution; use super::storage::{self, Result, fail}; use super::{ENTRY_LIMIT, Entry, RECORD_LIMIT, Store, Tree, now}; use cap_fs_ext::DirExt; @@ -19,29 +20,45 @@ pub(super) struct Change { } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct Journal { +pub(super) struct Journal { // Version 3 adds prepared native file/parent ownership. Version 2 binds // only the worker and cannot acquire that historical proof retroactively. - version: u32, - location: std::path::PathBuf, - root: String, - created: u64, - completed: bool, - changes: Vec, - applied: Vec, + pub version: u32, + pub location: std::path::PathBuf, + pub root: String, + pub created: u64, + pub completed: bool, + pub changes: Vec, + pub applied: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] - origin: Option, + pub origin: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] - acknowledged: bool, + pub acknowledged: bool, #[serde(default, skip_serializing_if = "Option::is_none")] - ownership: Option, + pub ownership: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RestoreContext { + pub target: String, + pub recorded: String, + pub expected: String, + pub exclusions: Vec, } impl Store { pub(super) fn ensure_recovered(&self) -> Result<()> { for id in storage::names(&self.journals, ENTRY_LIMIT)? { let (_, record) = self.journal(&id)?; - if !record.completed || (record.origin.is_some() && !record.acknowledged) { + if !record.completed + || ((record.origin.is_some() || record.resolution.is_some()) + && !record.acknowledged) + { return Err( "Snapshot restore requires recovery; private journal and preimages retained" .into(), @@ -51,7 +68,7 @@ impl Store { Ok(()) } - fn journal(&self, id: &str) -> Result<(cap_std::fs::Dir, Journal)> { + pub(super) fn journal(&self, id: &str) -> Result<(cap_std::fs::Dir, Journal)> { let dir = self.journals.open_dir_nofollow(id).map_err(fail)?; let record: Journal = serde_json::from_slice(&storage::read(&dir, "intent.json", RECORD_LIMIT)?) @@ -80,6 +97,19 @@ impl Store { if let Some(ownership) = &record.ownership { ownership.validate(&record.changes)?; } + if let Some(resolution) = &record.resolution { + resolution.validate(id, &record)?; + } + if let Some(context) = &record.context { + for id in [&context.target, &context.recorded, &context.expected] { + if !id.strip_prefix("copy:").is_some_and(storage::valid_digest) { + return Err("Invalid recovery snapshot context".into()); + } + } + if context.exclusions.len() > ENTRY_LIMIT * 3 { + return Err("Recovery exclusion inventory exceeds limit".into()); + } + } if let Some(origin) = &record.origin { let binding = &origin.binding; if origin.version != 1 @@ -121,7 +151,13 @@ impl Store { recorded: &str, expected: &str, paths: Option<&[String]>, - ) -> std::result::Result<(Tree, Vec), RestoreError> { + ) -> std::result::Result<(Tree, Vec, RestoreContext), RestoreError> { + let mut context = RestoreContext { + target: target.into(), + recorded: recorded.into(), + expected: expected.into(), + exclusions: vec![], + }; self.ensure_recovered().map_err(RestoreError::Failed)?; let before = self.capture().map_err(RestoreError::Failed)?; if self @@ -134,6 +170,15 @@ impl Store { } let target = self.load(target).map_err(RestoreError::Failed)?; let recorded = self.load(recorded).map_err(RestoreError::Failed)?; + let mut exclusions = std::collections::BTreeSet::new(); + for (label, tree) in [ + ("current", &before), + ("target", &target), + ("recorded", &recorded), + ] { + exclusions.extend(tree.skipped.iter().map(|path| format!("{label}: {path}"))); + } + context.exclusions = exclusions.into_iter().collect(); let mut changes = vec![]; let mut conflicts = vec![]; for path in super::changes(&recorded, &target) { @@ -198,7 +243,7 @@ impl Store { if !conflicts.is_empty() { return Err(RestoreError::Conflict(conflicts)); } - Ok((before, changes)) + Ok((before, changes, context)) } fn merged( &self, @@ -239,7 +284,7 @@ impl Store { expected: &str, paths: Option<&[String]>, ) -> std::result::Result, RestoreError> { - let (_, changes) = self.plan(target, recorded, expected, paths)?; + let (_, changes, _) = self.plan(target, recorded, expected, paths)?; changes .iter() .map(|c| { @@ -266,22 +311,32 @@ impl Store { expected: &str, paths: Option<&[String]>, authorize: &(dyn Fn() -> Result<()> + Send + Sync), - mut worker: Option<&mut SnapshotWork<'_>>, + worker: Option<&mut SnapshotWork<'_>>, ) -> std::result::Result, RestoreError> { authorize().map_err(RestoreError::Failed)?; if let Some(worker) = worker.as_ref() { worker.authorize().map_err(RestoreError::Failed)?; } - let (_, changes) = self.plan(target, recorded, expected, paths)?; + let (_, changes, context) = self.plan(target, recorded, expected, paths)?; if changes.is_empty() { return Ok(vec![]); } let id = format!("restore_{}", ulid::Ulid::new()); let mut journal = self - .prepare_journal(changes, &id, worker.as_deref()) + .prepare_journal(changes, &id, worker.as_deref(), Some(context), None) .map_err(RestoreError::Failed)?; + self.execute_journal(&id, &mut journal, authorize, worker) + } + + pub(super) fn execute_journal( + &self, + id: &str, + journal: &mut Journal, + authorize: &(dyn Fn() -> Result<()> + Send + Sync), + mut worker: Option<&mut SnapshotWork<'_>>, + ) -> std::result::Result, RestoreError> { // An unpublished/incomplete directory also remains recovery evidence. - let dir = storage::private_child(&self.journals, &id).map_err(RestoreError::Failed)?; + let dir = storage::private_child(&self.journals, id).map_err(RestoreError::Failed)?; let origin = worker .as_mut() .map(|worker| { @@ -289,32 +344,62 @@ impl Store { directory: self.root.clone(), root_identity: self.root_id.clone(), namespace_identity: storage::dir_identity(&self.journals)?, - journal_id: id.clone(), + journal_id: id.to_owned(), journal_identity: storage::dir_identity(&dir)?, + lock_identity: Some(self.lock_identity()?), }) }) .transpose() .map_err(RestoreError::Failed)?; journal.origin = origin; - Self::record(&dir, &journal).map_err(RestoreError::Failed)?; + Self::record(&dir, journal).map_err(RestoreError::Failed)?; + let preimages = self + .resolution_preimages(journal.resolution.as_ref()) + .map_err(RestoreError::Failed)?; for (i, change) in journal.changes.iter().enumerate() { if let Some(after) = &change.after { - let file = storage::create( - &dir, - &format!("stage_{i}"), - &self.bytes(after).map_err(RestoreError::Failed)?, - ) + let bytes = match &preimages { + Some(preimages) => preimages.bytes(self, &change.path, after), + None => self.bytes(after), + } .map_err(RestoreError::Failed)?; + let file = storage::create(&dir, &format!("stage_{i}"), &bytes) + .map_err(RestoreError::Failed)?; Self::stage_metadata(&file, after).map_err(RestoreError::Failed)?; } } storage::sync(&dir).map_err(RestoreError::Failed)?; journal.ownership = Some(Ownership::prepare(self, &dir, &journal.changes).map_err(RestoreError::Failed)?); - Self::record(&dir, &journal).map_err(RestoreError::Failed)?; + Self::record(&dir, journal).map_err(RestoreError::Failed)?; + if let Some(preimages) = &preimages { + preimages + .verify_review(self) + .map_err(RestoreError::Failed)?; + } for i in 0..journal.changes.len() { authorize().map_err(RestoreError::Failed)?; + let apply = |authorize: &dyn Fn() -> Result<()>| { + let verify = || { + authorize()?; + if let Some(preimages) = &preimages { + preimages.check_source(self, &journal.changes[i].path)?; + } + Ok(()) + }; + verify()?; + self.install( + &dir, + i, + &journal.changes[i], + journal + .ownership + .as_ref() + .ok_or("Missing restore ownership")?, + &verify, + ) + }; if let Some(worker) = worker.as_ref() { worker .effect(|| { @@ -324,47 +409,31 @@ impl Store { .as_ref() .ok_or("Missing snapshot worker origin")?, )?; - self.install( - &dir, - i, - &journal.changes[i], - journal - .ownership - .as_ref() - .ok_or("Missing restore ownership")?, - &|| worker.authorize(), - ) + apply(&|| worker.authorize()) }) .map_err(RestoreError::Failed)?; } else { - self.install( - &dir, - i, - &journal.changes[i], - journal - .ownership - .as_ref() - .ok_or_else(|| RestoreError::Failed("Missing restore ownership".into()))?, - authorize, - ) - .map_err(RestoreError::Failed)?; + apply(authorize).map_err(RestoreError::Failed)?; } journal.applied.push(journal.changes[i].path.clone()); - Self::record(&dir, &journal).map_err(RestoreError::Failed)?; + Self::record(&dir, journal).map_err(RestoreError::Failed)?; } + drop(preimages); self.verify_root().map_err(RestoreError::Failed)?; - self.verify_applied(&dir, &journal) + self.verify_applied(&dir, journal) .map_err(RestoreError::Failed)?; journal.completed = true; - Self::record(&dir, &journal).map_err(RestoreError::Failed)?; - Ok(journal.applied) + Self::record(&dir, journal).map_err(RestoreError::Failed)?; + Ok(journal.applied.clone()) } - fn prepare_journal( + pub(super) fn prepare_journal( &self, changes: Vec, id: &str, worker: Option<&SnapshotWork<'_>>, + context: Option, + mut resolution: Option, ) -> Result { let origin = worker .map(|worker| { @@ -375,10 +444,14 @@ impl Store { namespace_identity: storage::dir_identity(&self.journals)?, journal_id: id.to_owned(), journal_identity: "0".repeat(storage::IDENTITY_MAX_BYTES), + lock_identity: Some("0".repeat(storage::IDENTITY_MAX_BYTES)), })) }) .transpose()?; let ownership = Some(Ownership::budget(&changes)?); + if let Some(resolution) = &mut resolution { + resolution.outcome = Some(resolution.review.budget()); + } let mut journal = Journal { version: 3, location: self.root.clone(), @@ -392,6 +465,8 @@ impl Store { origin, acknowledged: worker.is_some(), ownership, + context, + resolution, }; let bytes = serde_json::to_vec(&journal).map_err(fail)?.len() as u64; if bytes > RECORD_LIMIT { @@ -416,6 +491,9 @@ impl Store { journal.origin = None; journal.acknowledged = false; journal.ownership = None; + if let Some(resolution) = &mut journal.resolution { + resolution.outcome = None; + } Ok(journal) } @@ -428,6 +506,7 @@ impl Store { return Err("Snapshot worker completion differs from its retained intent".into()); } self.verify_applied(&dir, &record)?; + self.verify_resolution(&dir, &record)?; Ok(()) } @@ -446,9 +525,15 @@ impl Store { Ok(()) } - fn verify_worker_namespace(&self, origin: &SnapshotOrigin) -> Result<()> { + pub(super) fn verify_worker_namespace(&self, origin: &SnapshotOrigin) -> Result<()> { self.verify_root()?; let binding = &origin.binding; + self.verify_lock_route( + binding + .lock_identity + .as_deref() + .ok_or("Legacy snapshot origin has no exact lock identity")?, + )?; if binding.directory != self.root || binding.root_identity != self.root_id || binding.namespace_identity != storage::dir_identity(&self.journals)? @@ -493,7 +578,7 @@ impl Store { .map_err(fail)?; storage::sync_file(file) } - fn record(dir: &cap_std::fs::Dir, journal: &Journal) -> Result<()> { + pub(super) fn record(dir: &cap_std::fs::Dir, journal: &Journal) -> Result<()> { let bytes = serde_json::to_vec(journal).map_err(fail)?; if bytes.len() as u64 > RECORD_LIMIT { return Err("Restore journal exceeds limit".into()); @@ -654,7 +739,7 @@ mod tests { assert!(old_estimate < 10000); assert!( store - .prepare_journal(changes.clone(), "restore_fixture", None) + .prepare_journal(changes.clone(), "restore_fixture", None, None, None) .is_err() ); assert!( @@ -664,7 +749,7 @@ mod tests { ); store.options.max_total = 100000; let journal = store - .prepare_journal(changes, "restore_fixture", None) + .prepare_journal(changes, "restore_fixture", None, None, None) .unwrap(); assert!(journal.applied.is_empty()); assert_eq!(journal.changes.len(), 1); @@ -701,6 +786,7 @@ mod tests { namespace_identity: storage::dir_identity(&store.journals).unwrap(), journal_id: id.into(), journal_identity: storage::dir_identity(&dir).unwrap(), + lock_identity: Some(store.lock_identity().unwrap()), }, }; Store::record( @@ -716,6 +802,8 @@ mod tests { origin: Some(origin.clone()), acknowledged: false, ownership: None, + context: None, + resolution: None, }, ) .unwrap(); diff --git a/crates/cyber-snapshot/src/copy/storage.rs b/crates/cyber-snapshot/src/copy/storage.rs index 7ab6ed5d7..2c213611b 100644 --- a/crates/cyber-snapshot/src/copy/storage.rs +++ b/crates/cyber-snapshot/src/copy/storage.rs @@ -164,12 +164,20 @@ pub fn lock(dir: &Dir, create: bool) -> Result { .mode(0o600) .follow(FollowSymlinks::No); let file = dir.open_with("lock", &options).map_err(fail)?.into_std(); + validate_lock(&file)?; + acquire_lock(&file)?; + if create { + sync_file(&file)?; + sync(dir)?; + } + Ok(file) +} +pub fn validate_lock(file: &File) -> Result<()> { let m = file.metadata().map_err(fail)?; if !m.is_file() || m.nlink() != 1 || m.mode() & 0o077 != 0 { return Err("Unsafe snapshot lock".into()); } - acquire_lock(&file)?; - Ok(file) + Ok(()) } fn acquire_lock(file: &File) -> Result<()> { let deadline = std::time::Instant::now() + std::time::Duration::from_millis(250); diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 366f7bd51..dd40af192 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -153,6 +153,43 @@ impl GitSnapshots { fn options(&self, location: &Path) -> (Vec, u64) { Self::options_value(&(self.config)(location)) } + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn copy_options(&self, directory: &str) -> copy::Options { + let config = (self.config)(Path::new(directory)); + let (ignore, max_file) = Self::options_value(&config); + let max_total = config + .pointer("/snapshots/fallback_max_bytes") + .and_then(Value::as_u64) + .unwrap_or(200 * 1024 * 1024); + copy::Options { + ignore, + max_file: max_file.min(max_total).min(64 * 1024 * 1024), + max_total, + private_roots: self.private_roots.clone(), + } + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn recovery_options(&self, directory: &str) -> copy::Options { + let mut options = self.copy_options(directory); + // Retained files remain inspectable after capture limits are lowered or + // snapshots are disabled. Resolution writes still obey the current cap. + options.max_file = 64 * 1024 * 1024; + options + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn recovery_location(&self, directory: &str) -> Result<(), String> { + let directory = Path::new(directory) + .canonicalize() + .map_err(|error| error.to_string())?; + if self + .private_roots + .iter() + .any(|root| directory.starts_with(root.canonicalize().unwrap_or_else(|_| root.clone()))) + { + return Err("Recovery Location is a protected private root".into()); + } + Ok(()) + } fn options_value(config: &Value) -> (Vec, u64) { let ignore = config .pointer("/snapshots/ignore") @@ -200,6 +237,79 @@ impl GitSnapshots { } impl Snapshots for GitSnapshots { + fn recovery_review( + &self, + directory: &str, + journal: &str, + action: cyber_server::runtime::SnapshotRecoveryAction, + ) -> BoxFuture<'_, Result> { + let directory = directory.to_owned(); + let journal = journal.to_owned(); + Box::pin(async move { + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = (directory, journal, action); + Err("Native recovery review is unavailable on this host".into()) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + self.recovery_location(&directory)?; + let data = self.data_dir.clone(); + let options = self.recovery_options(&directory); + tokio::task::spawn_blocking(move || { + let store = copy::Store::existing(&data, &directory, options)? + .ok_or("No retained file-copy namespace")?; + store.recovery_review(&journal, action) + }) + .await + .map_err(|error| error.to_string())? + } + }) + } + fn resolve_recovery( + &self, + directory: String, + request: cyber_server::runtime::SnapshotRecoveryConfirmation, + worker: cyber_server::runtime::SnapshotWorker, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = (directory, request, worker); + Err(RestoreError::Failed( + "Native recovery is unavailable on this host".into(), + )) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + self.recovery_location(&directory) + .map_err(RestoreError::Failed)?; + let options = self.recovery_options(&directory); + let data = self.data_dir.clone(); + let lock = cyber_server::workspace::mutation_lock(Path::new(&directory)) + .map_err(|error| RestoreError::Failed(error.to_string()))?; + let workspace = lock.lock_owned().await; + worker + .run(move |work| { + let store = copy::Store::recover_existing(&data, &directory, options) + .map_err(RestoreError::Failed)?; + let receipt = store.resolve_recovery(&request, &|| Ok(()), Some(work))?; + let origin = work.origin().cloned(); + let mut native = cyber_server::runtime::SnapshotNativeResult::new( + vec![], + Box::new(copy::WorkerGuard { + store, + workspace, + origin, + }), + ); + native.recovery = Some(receipt); + Ok(native) + }) + .await + } + }) + } fn owned_restore_available(&self, directory: &str) -> bool { cfg!(any(target_os = "macos", target_os = "linux")) && self.non_git(directory) } diff --git a/docs/rewind.md b/docs/rewind.md index af22d1fe5..97d825fba 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -208,7 +208,31 @@ evidence from matching content. Read-only native inspection never captures new objects or changes owner files. Created-parent permissions and other metadata are checked against preparation; existing owner-directory modes are preserved. -Managed keep/rollback resolution, lost-reply acknowledgement reconciliation and -reclamation of retained journals are not implemented yet. Do not edit records or -remove evidence to force a retry. Complete operator recovery and broader native -crash/platform acceptance remain unfinished. +Read-only recovery review is available through the owner API: + +```sh +cyber api POST /sessions//rewind/recovery/review \ + -d '{"journal_id":"restore_...","action":"rollback"}' +``` + +Use `"keep"` to review preserving the current files. SDK callers can use +`client.session.snapshotRecoveryReview(sessionID, { journal_id, action })`. +Review reports recorded before/after images, current and retained private images, +proposed changes, recorded created parents and retained exclusions. Missing +exclusion history is explicit. A foreign replacement cannot become a rollback +target merely because its bytes match. Live or already acknowledged activities, +foreign Sessions and unavailable native proof prohibit confirmation. Review does +not start inference, restore files, move the conversation or acknowledge activity. +The bounded review is limited to 16 MiB; the simpler journal catalog remains 4 MiB. + +New origins bind the exact private lock inode. A replacement lock file cannot +prove an old worker has exited; older origins without this proof stay inspectable +and fenced. Retained file-copy evidence remains inspectable when capture is +disabled or capture limits are lowered. Resolution writes still obey the cap. + +The native keep/rollback backend is implemented, but operator confirmation is not +yet exposed: dedicated runtime admission, durable native acknowledgement and +lost-reply reconciliation remain required. Created source directories and private +payloads remain protected for acknowledged safe cleanup. Do not edit records or +remove evidence to force a retry. Complete operator recovery, Git journaling, +reclamation and broader native crash/platform acceptance remain unfinished. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 8a43bcf25..3e63b7ce3 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -533,3 +533,77 @@ cross-filesystem native runs, the full workspace tests/Clippy, the runtime binar document suite and SDK TypeScript checks were not rerun in this checkpoint. No network DTO changed or SDK generation was required. M2–M8 and parent tasks remain open; neither issue is complete. + +## Recovery review and independent native resolution backend + +The previous goal turn made verified progress in `52147d8d`. This continuation +adds owner review at `POST /sessions/{id}/rewind/recovery/review` and generated SDK +`session.snapshotRecoveryReview`. Selection binds the journal/action, selected +Session creation, current canonical Location/checkout and exact native activity. +Live, missing, foreign, generic or already acknowledged activity cannot acquire a +confirmation from that review. It starts no inference and changes no transcript, +source file or native acknowledgement. Review includes current, recorded +before/after, retained private and proposed image metadata; recorded created +parents; retained exclusions or explicit unavailable history; and conflicts. Its +response is bounded at 16 MiB. The simpler catalog retains its 4 MiB limit. + +New intents retain the original snapshot selectors and bounded exclusion history. +The native backend fingerprints exact file identities/generations, parent +observations, immutable intent and action. Keep preserves reviewed ordinary owner +replacements without assigning them to the restore. Rollback requires strict +native ownership and stages from actual retained preimages, independently of +capture objects. Both publish a separate version-3 resolution intent and retain +actual displaced live inodes. A prior request ID is lookup-only, including an +incomplete outcome; different ownership/content refuses. A generation change after +preparation is not adopted as new source ownership. Whole-plan validation precedes +source effects and each file operation rechecks its reviewed source. Created +source directories remain protected for acknowledged safe cleanup. + +The new snapshot port returns an explicit filesystem recovery receipt. Ordinary +rewind consuming commit refuses that receipt rather than acknowledging only the +new worker and pretending the original activity was resolved. Dedicated recovery +admission, global durable request ownership, exact original/resolution writer +commit, acknowledgement reconciliation and operator apply routes/CLI/TUI are not +implemented by this backend. Neither native fence is cleared by file completion. + +Substantive code/security review found a replaceable-lock proof gap. New origins +bind the exact private lock inode and birth identity; new lock creation is flushed +before origin publication. Recovery opens an existing namespace and its existing +exclusive lock without creating missing evidence. The regression actually replaces +a live owner's lock and acquires the different inode, then proves native recovery +refuses that false exit proof. Origins without the binding remain inspectable and +fenced. Inspection ignores capture disable/lowered capture size while retaining a +64 MiB native file bound; resolution writes retain the configured total cap. +Retained file-copy journals are selected by provenance rather than current capture +enablement or later Git discovery. Git-native journals remain unimplemented. + +Review also corrected capture-object dependence, a private-preparation adoption +window, missing visible retained images, and stale native generations. Tests cover +whole-plan conflict refusal, keep after owner edits, partial resolution evidence, +duplicate request refusal without even calling authorization, quota failure and +retry after capacity is raised. Runtime metadata tests distinguish live/unknown/ +settled origins and refuse generic or mismatched activities. They are instrumented +attestation tests, not native consuming-transaction proof. + +Final local macOS snapshot tests passed 67 cases (32 unit, 18 file-copy, 17 Git); +the unsupported-platform binary selected zero. All 11 focused runtime snapshot +tests passed. All 13 real CLI branch/recovery cases passed; the final review case +was rerun after the last canonical-Location validation and passed. It verifies +Session ownership, acknowledgement limits, retained exclusion history, unchanged +files/messages and no additional scripted-provider inference. Affected server, +snapshot and CLI all-target Clippy with warnings denied passed. OpenAPI consistency +and SDK generation checks passed. SDK typecheck passed; SDK tests passed 129 cases +with ten configured process end-to-end cases skipped because `CYBER_BIN` was unset. +The new SDK review transport contract ran. Strict OpenSpec passed 117 items; lint +reported zero errors/25 warnings, inventory remained 53 capabilities/1030 +requirements, and formatting/diff checks passed. Native sandbox deprecation and +Darwin linker warnings were observed without attributing their origin to main. + +No external model provider was used. Linux/Windows, full workspace tests/Clippy, +TUI/App recovery acceptance, real SDK process acceptance, native resolution +process-kill cuts and paired runtime lost-reply/reconciliation were not run in this +checkpoint. The earlier storage kill tests still run but do not substitute for +these missing resolution/runtime cuts. Late-scope disposal and uncertainty-reason +requirements are recorded in tasks.md before enabling acknowledgement. The remote +refresh still reports main `db9e171e` and no open PRs. M2–M8 remain open; the full +#93/#92 goal remains active and neither issue is complete. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index b3644478b..88f5d532b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -27,6 +27,17 @@ replays the original restore or treats a missing `applied` entry as no effect. Unsupported metadata, aliases, cross-filesystem staging, ambiguous/malformed records or changed Location identity refuse rather than weakening the native contract. +Keep may review a later ordinary owner replacement as the current image; preserving +it does not assign that inode to the failed restore. Rollback still requires the +original source/private ownership pairing. Both actions bind exact current images +and parent observations into the review and refuse stale confirmation. A resolution +uses a distinct native intent, retaining its displaced live inodes and a reference +to the original immutable intent. Retry of an existing request is lookup-only even +when its outcome is partial. Original and resolution evidence remain fenced until +the consuming runtime transaction acknowledges the exact workers. Empty created +source directories remain protected until acknowledged, ownership-checked cleanup; +file rollback does not recursively remove owner directories. + ## Bind evidence to a worker, not merely a Session New managed restore intents record an immutable origin: Session creation identity, @@ -34,6 +45,10 @@ canonical/native Location identity, native activity ID, checkout binding, journa namespace identity and version. The runtime marks the activity as a snapshot filesystem-only worker before it can launch. Snapshot callbacks cannot label a generic shell/process activity as filesystem-only. +New origins also bind the exact private lock inode, including its creation +identity. Acquiring a replacement lock file cannot prove that an old worker has +released its original descriptor. Older origins without this binding stay +inspectable and fenced; no retrospective lock proof is invented. The snapshot-worker compatibility migration records the new event semantics even without new tables. Older migration catalogs must refuse mutation rather than diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 256999286..5bfd36706 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -120,6 +120,21 @@ - **AND** an equal-byte replacement with a foreign native identity, a changed parent or alias refuses ownership proof; older journals without prepared evidence remain readable without gaining rollback authority - **AND** original preimages and private preparations are flushed before prepared evidence is published, created-parent metadata changes refuse, and authority is rechecked before each parent publication and file operation +#### Scenario: Owner reviews keep and rollback +- **WHEN** the authenticated owner reviews a retained restore +- **THEN** review reports exact native origin, current/proposed images, conflicts and retained exclusions or explicit unavailable exclusion history, without restoring, acknowledging activity or starting inference +- **AND** keep preserves a reviewed later ordinary owner replacement without claiming it belongs to the restore, while rollback refuses a foreign replacement or missing native proof + +#### Scenario: Replacing the private lock cannot manufacture worker exit +- **WHEN** the private lock file is replaced while an old worker retains its original descriptor +- **THEN** recovery refuses the changed lock identity even if a different lock inode can be acquired +- **AND** origins without an exact native lock binding remain inspectable and fenced rather than receiving retrospective acknowledgement authority + +#### Scenario: Recovery request never replays filesystem resolution +- **WHEN** a fingerprint-confirmed resolution has published a distinct durable request intent +- **THEN** later requests with its ID look up that original result or incomplete outcome and never dispatch source changes again +- **AND** changed request content refuses, preimages and directory ownership evidence remain retained, and filesystem completion alone does not settle original/native fences + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index c7376defd..496ccc8a0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -42,6 +42,25 @@ regressions. Native origin validation and resolution request ownership are not implemented by that primitive; M5 remains open until the runtime uses it with exact native proof and actual restart/retry acceptance. +M4/M7 continuation: the read-only recovery review API and native file-copy +resolution backend now exist. The backend writes a separate intent and retains +actual displaced live inodes; existing request IDs are lookup-only. This is not +operator apply acceptance. Dedicated recovery admission, global durable request +ownership and the exact original/resolution writer pair remain required. Ordinary +rewind commit explicitly refuses a recovery receipt. No native fence is cleared by +backend filesystem completion. Created source directories remain protected for +acknowledged, ownership-checked cleanup. + +Before enabling M5, cover late original-scope disposal after native store-lock +release: an `Expected::Any` Drop append must not resurrect an acknowledged activity +or overwrite its durable receipt. Preserve opaque/generic Location uncertainty; +the current shared `location_uncertain` boolean is not proof that every reason +belongs to the resolved snapshot. Reconciliation must distinguish a durable writer +acknowledgement with missing journal acknowledgement from an uncommitted filesystem +outcome, and must not revalidate or replay later owner edits as the old request. +Unprepared reservation/origin/intent cuts still require exact resource ownership +proof; missing records alone do not license acknowledgement. + - [ ] M6 Publish journal acknowledgement only after observed durable Store settlement. Add reference/reader/lease-aware eligibility and durable two-phase reclamation with retained retry receipts. Test cap pressure, disabled/zero retention, incomplete reference views and crashes at writer/journal/reclaim boundaries without deleting unresolved evidence. - [ ] M7 Add owner recovery review/confirm/lookup HTTP routes, typed CLI and SDK operations, and TUI recovery entry from rewind unavailability. Regenerate OpenAPI/SDK/command artifacts; exercise the real clients with no inference or transcript movement and with stale/foreign request refusal. - [ ] M8 Run process-kill cuts before/after intent, stage preparation, every exchange, progress, filesystem completion, writer commit, journal acknowledgement and reclaim. Obtain Linux x64/ARM and macOS Intel/ARM positives/denials, explicit Windows results/limits and shared Git/binary regressions; update evidence and perform substantive code/security review before accepting parent tasks. @@ -70,6 +89,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - `cyber-server/src/runtime/activity.rs` owns native activity records; `binary_documents.rs` retains a real synchronous worker's resources and `binary_recovery.rs` demonstrates document-specific recovery proof. Snapshot workers require their own origin/proof rather than calling binary recovery. - `cyber-server/src/runtime/host.rs::Snapshots` currently exposes restore and read-only recovery inspection; `rewind_preview.rs` owns reviewed confirmation. A dedicated owner recovery admission must work while ordinary admission remains fenced. - `cyber-snapshot/src/copy/restore.rs` reads legacy version-1/2 intents and writes version-3 native ownership records. `copy/ownership.rs` prepares file/parent evidence and inspects actual slots; `copy/storage.rs` owns private descriptor-relative I/O. `restore.rs` and `git_history.rs` cover Git restore and historical exclusions. Keep historical evidence readable as the new protocol is added. +- `cyber-snapshot/src/copy/resolution.rs` freezes native observations, reads actual retained preimages, publishes independent keep/rollback intents and rejects request replay. Runtime `snapshot_recovery.rs` attests the selected Session/activity for read-only review; its consuming admission/commit is still required. Origins now bind the exact private lock inode; older origins without that proof remain inspectable and fenced. - `cyber-store/src/store.rs::append_checked` currently appends one aggregate. Separate independent appends cannot establish atomic acknowledgement plus resolution ownership. ## 4. Rules and context inspection diff --git a/sdk/openapi.json b/sdk/openapi.json index dfd7167f9..c036106da 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7061,6 +7061,298 @@ ], "description": "Retained filesystem evidence; completion does not acknowledge a Session/native receipt." }, + "SnapshotRecoverySelection": { + "type": "object", + "additionalProperties": false, + "properties": { + "journal_id": { + "type": "string" + }, + "action": { + "$ref": "#/components/schemas/SnapshotRecoveryAction" + } + }, + "required": [ + "journal_id", + "action" + ] + }, + "SnapshotRecoveryAction": { + "type": "string", + "enum": [ + "keep", + "rollback" + ] + }, + "Data_SnapshotRecoveryReview": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/SnapshotRecoveryReview" + } + }, + "required": [ + "data" + ] + }, + "SnapshotRecoveryReview": { + "type": "object", + "properties": { + "journal_id": { + "type": "string" + }, + "action": { + "$ref": "#/components/schemas/SnapshotRecoveryAction" + }, + "origin": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + { + "type": "null" + } + ] + }, + "filesystem_complete": { + "type": "boolean" + }, + "acknowledged": { + "type": "boolean" + }, + "confirmation": { + "type": [ + "string", + "null" + ], + "description": "Absent when conflicts or unavailable historical evidence prohibit confirmation." + }, + "paths": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SnapshotRecoveryPath" + } + }, + "exclusions": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "None means historical exclusions were not retained; not an empty exclusion set." + }, + "recorded_created_parents": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "Recorded created parents remain protected until acknowledged safe cleanup." + }, + "conflicts": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "journal_id", + "action", + "filesystem_complete", + "acknowledged", + "paths", + "conflicts" + ] + }, + "SnapshotOrigin": { + "type": "object", + "additionalProperties": false, + "properties": { + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "session_id": { + "type": "string" + }, + "session_created_id": { + "type": "string" + }, + "activity_id": { + "type": "string" + }, + "worktree_id": { + "type": [ + "string", + "null" + ] + }, + "binding": { + "$ref": "#/components/schemas/SnapshotBinding" + } + }, + "required": [ + "version", + "session_id", + "session_created_id", + "activity_id", + "binding" + ] + }, + "SnapshotBinding": { + "type": "object", + "additionalProperties": false, + "properties": { + "directory": { + "type": "string" + }, + "root_identity": { + "type": "string" + }, + "namespace_identity": { + "type": "string" + }, + "journal_id": { + "type": "string" + }, + "journal_identity": { + "type": "string" + }, + "lock_identity": { + "type": [ + "string", + "null" + ], + "description": "Exact private lock inode. Legacy origins do not acquire this proof retroactively." + } + }, + "required": [ + "directory", + "root_identity", + "namespace_identity", + "journal_id", + "journal_identity" + ] + }, + "SnapshotRecoveryPath": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "state": { + "type": "string", + "description": "`applied`, `unapplied`, or `foreign`; foreign files are never rollback targets." + }, + "current": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRecoveryImage" + }, + { + "type": "null" + } + ] + }, + "recorded_before": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRecoveryImage" + }, + { + "type": "null" + } + ] + }, + "recorded_after": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRecoveryImage" + }, + { + "type": "null" + } + ] + }, + "retained": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRecoveryImage" + }, + { + "type": "null" + } + ], + "description": "Current private slot: staged target or displaced preimage, without inventing ownership." + }, + "proposed": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRecoveryImage" + }, + { + "type": "null" + } + ] + }, + "changes": { + "type": "boolean" + } + }, + "required": [ + "path", + "state", + "changes" + ] + }, + "SnapshotRecoveryImage": { + "type": "object", + "properties": { + "hash": { + "type": "string" + }, + "bytes": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "mode": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "uid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "gid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "native_identity": { + "type": [ + "string", + "null" + ], + "description": "Expected images and proposed copies need not identify a live inode." + } + }, + "required": [ + "hash", + "bytes", + "mode", + "uid", + "gid" + ] + }, "RewindSelection": { "type": "object", "additionalProperties": false, @@ -23921,6 +24213,65 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/recovery/review": { + "post": { + "operationId": "v1.session.snapshotRecoveryReview", + "summary": "Review retained snapshot effects without restoring, acknowledging native activity or starting inference", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotRecoveryReview" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SnapshotRecoverySelection" + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/preview": { "post": { "operationId": "v1.session.rewindPreview", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index 4248d0557..86967e94f 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1559,6 +1559,12 @@ export interface Operations { body: undefined; response: S.SnapshotRecovery[]; }; + "v1.session.snapshotRecoveryReview": { + path: { sessionID: string }; + query: Record; + body: S.SnapshotRecoverySelection; + response: S.SnapshotRecoveryReview; + }; "v1.session.startDelegation": { path: { sessionID: string; requestID: string }; query: Record; @@ -4644,6 +4650,18 @@ export const operations = { unwrap: true, stream: false, }, + "v1.session.snapshotRecoveryReview": { + tag: "session", + name: "snapshotRecoveryReview", + method: "POST", + path: "/api/v1/sessions/{sessionID}/rewind/recovery/review", + pathParams: ["sessionID"], + query: [], + body: "required", + located: false, + unwrap: true, + stream: false, + }, "v1.session.startDelegation": { tag: "session", name: "startDelegation", @@ -6037,6 +6055,9 @@ export function createGroups(caller: Caller) { /** Inspect retained filesystem restore journals without acknowledging or replaying native effects (`GET /api/v1/sessions/{sessionID}/rewind/recovery`) */ snapshotRecovery: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecovery", { path: { sessionID } }, options), + /** Review retained snapshot effects without restoring, acknowledging native activity or starting inference (`POST /api/v1/sessions/{sessionID}/rewind/recovery/review`) */ + snapshotRecoveryReview: (sessionID: string, body: Operations["v1.session.snapshotRecoveryReview"]["body"], options?: RequestOptions): Promise => + caller.call("v1.session.snapshotRecoveryReview", { path: { sessionID }, body }, options), /** Durably reserve a named user delegation request (`POST /api/v1/sessions/{sessionID}/delegations/{requestID}`) */ startDelegation: (sessionID: string, requestID: string, body: Operations["v1.session.startDelegation"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.startDelegation", { path: { sessionID, requestID }, body }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 14d0358dd..0507c1126 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -1106,6 +1106,10 @@ export type Data_SideAnswer = { data: SideAnswer; }; +export type Data_SnapshotRecoveryReview = { + data: SnapshotRecoveryReview; +}; + export type Data_SubtreeReopenReport = { data: SubtreeReopenReport; }; @@ -4685,6 +4689,25 @@ export type SkillInfo = { /** The scope a skill was discovered in. */ export type SkillOrigin = "builtin" | "user" | "project" | "config" | "plugin"; +export type SnapshotBinding = { + directory: string; + root_identity: string; + namespace_identity: string; + journal_id: string; + journal_identity: string; + /** Exact private lock inode. Legacy origins do not acquire this proof retroactively. */ + lock_identity?: string | null; +}; + +export type SnapshotOrigin = { + version: number; + session_id: string; + session_created_id: string; + activity_id: string; + worktree_id?: string | null; + binding: SnapshotBinding; +}; + /** Retained filesystem evidence; completion does not acknowledge a Session/native receipt. */ export type SnapshotRecovery = { id: string; @@ -4695,6 +4718,52 @@ export type SnapshotRecovery = { applied: string[]; }; +export type SnapshotRecoveryAction = "keep" | "rollback"; + +export type SnapshotRecoveryImage = { + hash: string; + bytes: number; + mode: number; + uid: number; + gid: number; + /** Expected images and proposed copies need not identify a live inode. */ + native_identity?: string | null; +}; + +export type SnapshotRecoveryPath = { + path: string; + /** `applied`, `unapplied`, or `foreign`; foreign files are never rollback targets. */ + state: string; + current?: SnapshotRecoveryImage | null; + recorded_before?: SnapshotRecoveryImage | null; + recorded_after?: SnapshotRecoveryImage | null; + /** Current private slot: staged target or displaced preimage, without inventing ownership. */ + retained?: SnapshotRecoveryImage | null; + proposed?: SnapshotRecoveryImage | null; + changes: boolean; +}; + +export type SnapshotRecoveryReview = { + journal_id: string; + action: SnapshotRecoveryAction; + origin?: SnapshotOrigin | null; + filesystem_complete: boolean; + acknowledged: boolean; + /** Absent when conflicts or unavailable historical evidence prohibit confirmation. */ + confirmation?: string | null; + paths: SnapshotRecoveryPath[]; + /** None means historical exclusions were not retained; not an empty exclusion set. */ + exclusions?: string[] | null; + /** Recorded created parents remain protected until acknowledged safe cleanup. */ + recorded_created_parents?: string[] | null; + conflicts: string[]; +}; + +export type SnapshotRecoverySelection = { + journal_id: string; + action: SnapshotRecoveryAction; +}; + export type SolveCandidateEvidence = { candidate_id: string; label?: string | null; diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index 36fb9ffac..9f4a618dd 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -16,3 +16,18 @@ test("reviewed rewind operations preserve the selected scope and fingerprint", a assert.equal(calls[1]!.url.pathname, "/api/v1/sessions/ses_one/rewind/confirm"); assert.deepEqual(JSON.parse(calls[1]!.body!), { ...selection, confirmation: fingerprint }); }); + +test("snapshot recovery review preserves owner scope and honest acknowledgement limits", async () => { + const review = { journal_id: "restore_one", action: "keep", origin: null, filesystem_complete: true, + acknowledged: false, confirmation: null, paths: [], exclusions: null, recorded_created_parents: null, + conflicts: ["Legacy intent has no exact native worker origin"] }; + const { client, calls } = mockClient(() => json(200, { data: review })); + const result = await client.session.snapshotRecoveryReview("ses_owner", { journal_id: "restore_one", action: "keep" }); + assert.deepEqual(result, review); + assert.equal(result.acknowledged, false); + assert.equal(result.confirmation, null); + assert.equal(calls.length, 1); + assert.equal(calls[0]!.method, "POST"); + assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/review"); + assert.deepEqual(JSON.parse(calls[0]!.body!), { journal_id: "restore_one", action: "keep" }); +}); From 93b4b613c0aa5b5eb3102cb912de536a919d3a3f Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 18:10:32 -0400 Subject: [PATCH 05/87] Preserve durable native acknowledgements during late scope disposal --- crates/cyber-server/src/runtime/activity.rs | 59 +++++++++-- .../src/runtime/snapshot_work_tests.rs | 99 +++++++++++++++++++ .../implementation.md | 38 +++++++ .../complete-daily-coding-surfaces/tasks.md | 8 +- 4 files changed, 191 insertions(+), 13 deletions(-) diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index d52feb123..93ac84a1e 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -1,6 +1,7 @@ //! Durable idle-operation ownership; disposal is not native acknowledgement. use super::{AdmissionAuthority, Handle, Inner, Runtime, RuntimeError}; use cyber_store::{Expected, NewEvent, Store}; +use rusqlite::OptionalExtension; use serde::{Deserialize, Serialize}; use std::sync::{Arc, PoisonError, Weak, atomic::Ordering}; use tokio_util::sync::CancellationToken; @@ -227,32 +228,70 @@ impl Scope { ) .await } + fn dispose(&self, record: Record) -> Result { + let id = self.id.clone(); + let original = self.record.clone(); + let sequence = if self.launched { 1 } else { 0 }; + let (_, uncertain) = self.store.append_checked(&self.id, Expected::Any, move |tx| { + let latest = tx.query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&id], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?)), + ).optional()?; + let Some((seq, kind, data)) = latest else { + return Err(cyber_store::StoreError::InvalidAppend("Native activity disappeared before disposal".into())); + }; + let current: Record = serde_json::from_str(&data).map_err(|error| + cyber_store::StoreError::CorruptEvent { id: id.clone(), reason: error.to_string() })?; + // An unsuccessful launch may have set the in-memory origin without + // publishing it. Its durable reservation must still have no origin. + let origin = if sequence == 1 { original.snapshot_origin.as_ref() } else { None }; + if kind != CHANGED || current.session_id != original.session_id + || current.binary_document != original.binary_document + || current.snapshot_worker != original.snapshot_worker + || current.snapshot_origin.as_ref() != origin + { + return Err(cyber_store::StoreError::InvalidAppend("Native activity ownership changed before disposal".into())); + } + // Recovery may have committed after the native lock was released but + // before this scope unwound. Preserve the complete durable receipt. + if current.status == "settled" { + return Ok((vec![], false)); + } + if seq != sequence { + return Err(cyber_store::StoreError::Concurrency { aggregate: id, expected: sequence, actual: seq }); + } + let uncertain = record.status == "unknown"; + Ok((vec![change(&record)], uncertain)) + })?; + Ok(uncertain) + } } impl Drop for Scope { fn drop(&mut self) { self.control.stop.cancel(); if !self.settled { let filesystem_only = self.record.binary_document.is_some(); - if self.launched && !filesystem_only { - self.handle.location_uncertain.store(true, Ordering::SeqCst); - } // This trusted scope owns only synchronous filesystem calls. Once // its actual worker unwinds, it has no escaped task/process to settle; // possible file effects remain in the separate document intent. - self.record.status = if self.launched && !filesystem_only { + let mut disposed = self.record.clone(); + disposed.snapshot_origin = self.snapshot_origin().cloned(); + disposed.status = if self.launched && !filesystem_only { "unknown" } else { "settled" } .into(); if filesystem_only { - self.record.phase = "filesystem_worker_disposed_query_document".into(); + disposed.phase = "filesystem_worker_disposed_query_document".into(); } - self.record.admission_bindings = None; - if let Err(error) = - self.store - .append(&self.id, Expected::Any, vec![change(&self.record)]) - { + disposed.admission_bindings = None; + let result = self.dispose(disposed); + if self.launched && !filesystem_only && result.as_ref().copied().unwrap_or(true) { + self.handle.location_uncertain.store(true, Ordering::SeqCst); + } + if let Err(error) = result { cyber_core::log::error( "activity", &error.to_string(), diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index c5822f6d0..c76eb9368 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -60,6 +60,104 @@ fn restart(runtime: &Runtime, root: &std::path::Path) -> Runtime { }) } +#[tokio::test] +async fn late_snapshot_disposal_preserves_durable_acknowledgement_and_unrelated_uncertainty() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + for unrelated_uncertainty in [false, true] { + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + let completed = owner + .run(move |work| { + let origin = work.begin(native_binding).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: Arc::new(AtomicUsize::new(0)), + }), + )) + }) + .await + .unwrap(); + let id = completed.worker.scope.id().to_owned(); + let mut receipt = latest(&runtime, &session); + receipt["status"] = "settled".into(); + receipt["phase"] = "snapshot_recovery_committed".into(); + receipt["resolution_receipt"] = serde_json::json!({"request_id":"retained_request"}); + runtime + .inner + .store + .append( + &id, + cyber_store::Expected::Seq(1), + vec![NewEvent::new(activity::CHANGED, receipt.clone())], + ) + .unwrap(); + let sequence = runtime.inner.store.aggregate_seq(&id).unwrap(); + let notifications = runtime.inner.store.subscribe(); + let handle = runtime.inner.handle(&session).await.unwrap(); + handle + .location_uncertain + .store(unrelated_uncertainty, Ordering::SeqCst); + drop(completed); + assert!(notifications.try_recv().is_err()); + assert_eq!(runtime.inner.store.aggregate_seq(&id).unwrap(), sequence); + assert_eq!(latest(&runtime, &session), receipt); + assert_eq!( + handle.location_uncertain.load(Ordering::SeqCst), + unrelated_uncertainty + ); + } + runtime.shutdown().await; +} + +#[tokio::test] +async fn late_snapshot_disposal_refuses_changed_native_ownership() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + let completed = owner + .run(move |work| { + let origin = work.begin(native_binding).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: Arc::new(AtomicUsize::new(0)), + }), + )) + }) + .await + .unwrap(); + let id = completed.worker.scope.id().to_owned(); + let mut foreign = latest(&runtime, &session); + foreign["snapshot_origin"]["binding"]["journal_id"] = "restore_foreign".into(); + runtime + .inner + .store + .append( + &id, + cyber_store::Expected::Seq(1), + vec![NewEvent::new(activity::CHANGED, foreign.clone())], + ) + .unwrap(); + let notifications = runtime.inner.store.subscribe(); + drop(completed); + assert!(notifications.try_recv().is_err()); + assert_eq!(runtime.inner.store.aggregate_seq(&id).unwrap(), Some(2)); + assert_eq!(latest(&runtime, &session), foreign); + assert!( + runtime + .inner + .handle(&session) + .await + .unwrap() + .location_uncertain + .load(Ordering::SeqCst) + ); + runtime.shutdown().await; +} + #[tokio::test] async fn ordinary_restore_commit_cannot_consume_a_recovery_receipt_or_acknowledge_it() { let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; @@ -153,6 +251,7 @@ async fn rejected_origin_append_cannot_authorize_source_effects_even_if_the_adap assert!(result.is_err()); runtime.inner.store.set_append_fault(None); assert_eq!(latest(&runtime, &session)["status"], "settled"); + assert!(latest(&runtime, &session)["snapshot_origin"].is_null()); runtime.shutdown().await; } diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 3e63b7ce3..e07ee43b8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -607,3 +607,41 @@ these missing resolution/runtime cuts. Late-scope disposal and uncertainty-reaso requirements are recorded in tasks.md before enabling acknowledgement. The remote refresh still reports main `db9e171e` and no open PRs. M2–M8 remain open; the full #93/#92 goal remains active and neither issue is complete. + +### Late native-scope disposal safeguard + +Scope disposal now reads the latest activity in the same Store writer transaction +as its conditional append. It checks Session, worker kind, document/origin and +expected reservation/launch sequence. An already-settled exact owner is a no-op: +its complete receipt, event sequence and notification stream are preserved. A +changed owner or sequence refuses the append and retains conservative uncertainty. +Rejected origin publication also no longer leaks the attempted in-memory origin +into the reservation's disposal event. + +Two new instrumented runtime regressions cover late disposal after an independently +committed acknowledgement (with and without an unrelated uncertainty flag), and +changed journal ownership. They check unchanged durable payload/sequence and no +notification from either the no-op or rejected transaction. The existing rejected +origin regression now verifies that no origin was manufactured by disposal. These +use the real Store and worker lifetime but inject the acknowledgement event; they +do not prove the still-unimplemented consuming recovery commit or a native crash. + +Final local macOS server unit tests passed 396 cases, with one ignored subprocess +fixture (`database_lock_probe`, invoked explicitly by its parent regressions). +Server all-target Clippy with warnings denied passed. Strict OpenSpec validation +passed all 117 items. Formatting and diff checks passed. + +The shared idle-cancellation and Location integration binaries also passed (four +and thirteen tests respectively), exercising acknowledged versus unknown disposal, +shutdown, cancellation timeouts, worker panic and retained resource ownership. + +The code-review skill was applied to the changed disposal path and tests, including +writer serialization, exact ownership, corrupt/missing evidence refusal, no-op +notifications and shared binary/generic-worker behavior. Review caught and fixed +the rejected-origin serialization issue. No new public operation or acknowledgement +authority is added. Opaque Location uncertainty is never cleared here. The wider M5 +work must still distinguish snapshot uncertainty from other reasons and coordinate +its in-memory state with recovery acknowledgement, including acknowledgement that +follows a successful unknown-disposal append. Git, runtime request ownership, +reconciliation, cleanup and native/client acceptance remain open. No Linux/Windows, +full-workspace or real recovery operator acceptance was run for this safeguard. diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 496ccc8a0..2b5038896 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -51,9 +51,11 @@ rewind commit explicitly refuses a recovery receipt. No native fence is cleared backend filesystem completion. Created source directories remain protected for acknowledged, ownership-checked cleanup. -Before enabling M5, cover late original-scope disposal after native store-lock -release: an `Expected::Any` Drop append must not resurrect an acknowledged activity -or overwrite its durable receipt. Preserve opaque/generic Location uncertainty; +Late original-scope disposal now checks durable identity and status within the +writer transaction after native store-lock release. It preserves an already +acknowledged activity and its complete receipt; changed ownership/sequence refuses +the append. This does not implement recovery acknowledgement. Before enabling M5, +preserve opaque/generic Location uncertainty; the current shared `location_uncertain` boolean is not proof that every reason belongs to the resolved snapshot. Reconciliation must distinguish a durable writer acknowledgement with missing journal acknowledgement from an uncommitted filesystem From fd732d814b7aca67393089ca174584c6e3d9d124 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 18:19:33 -0400 Subject: [PATCH 06/87] Track snapshot uncertainty by exact native activity owner --- crates/cyber-server/src/runtime/activity.rs | 18 ++- .../cyber-server/src/runtime/auto_override.rs | 4 +- .../src/runtime/binary_documents_tests.rs | 12 +- .../cyber-server/src/runtime/continuation.rs | 2 +- crates/cyber-server/src/runtime/location.rs | 4 +- .../src/runtime/location_uncertainty.rs | 65 +++++++++++ crates/cyber-server/src/runtime/mod.rs | 11 +- crates/cyber-server/src/runtime/names.rs | 10 +- .../src/runtime/rewind_preview.rs | 3 +- .../cyber-server/src/runtime/snapshot_work.rs | 4 +- .../src/runtime/snapshot_work_tests.rs | 103 +++++++++++++++-- .../src/runtime/subtree_reopen.rs | 3 +- .../cyber-server/src/runtime/subtree_stop.rs | 4 +- .../src/runtime/worktree_events.rs | 6 +- crates/cyber-server/tests/rewind_preview.rs | 107 ++++++++++++++++++ .../implementation.md | 54 +++++++++ .../managed-recovery.md | 7 ++ .../complete-daily-coding-surfaces/tasks.md | 9 +- 18 files changed, 373 insertions(+), 53 deletions(-) create mode 100644 crates/cyber-server/src/runtime/location_uncertainty.rs diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 93ac84a1e..ba7e77d28 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -231,7 +231,14 @@ impl Scope { fn dispose(&self, record: Record) -> Result { let id = self.id.clone(); let original = self.record.clone(); + let handle = self.handle.clone(); let sequence = if self.launched { 1 } else { 0 }; + // Publish before enqueueing. Even an unavailable writer retains the exact + // owner; a later writer acknowledgement can clear it without racing a + // post-commit caller-side flag write. + if self.launched && original.snapshot_worker { + handle.mark_snapshot_uncertain(&id); + } let (_, uncertain) = self.store.append_checked(&self.id, Expected::Any, move |tx| { let latest = tx.query_row( "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", @@ -256,6 +263,9 @@ impl Scope { // Recovery may have committed after the native lock was released but // before this scope unwound. Preserve the complete durable receipt. if current.status == "settled" { + if current.snapshot_worker { + handle.acknowledge_snapshot_uncertainty(&id); + } return Ok((vec![], false)); } if seq != sequence { @@ -288,7 +298,11 @@ impl Drop for Scope { } disposed.admission_bindings = None; let result = self.dispose(disposed); - if self.launched && !filesystem_only && result.as_ref().copied().unwrap_or(true) { + if self.launched + && !filesystem_only + && !self.record.snapshot_worker + && result.as_ref().copied().unwrap_or(true) + { self.handle.location_uncertain.store(true, Ordering::SeqCst); } if let Err(error) = result { @@ -364,7 +378,7 @@ impl Inner { }, )); } - if handle.location_uncertain.load(Ordering::SeqCst) { + if handle.has_location_uncertainty() { return Err(RuntimeError::Invalid( "Idle operation did not acknowledge interruption; recovery is required".into(), )); diff --git a/crates/cyber-server/src/runtime/auto_override.rs b/crates/cyber-server/src/runtime/auto_override.rs index 675cde67a..293b2f775 100644 --- a/crates/cyber-server/src/runtime/auto_override.rs +++ b/crates/cyber-server/src/runtime/auto_override.rs @@ -369,9 +369,7 @@ impl Inner { record.status = "cancelled".into(); self.commit(handle, vec![change(&record)]).await?; } - if !handle - .location_uncertain - .load(std::sync::atomic::Ordering::SeqCst) + if !handle.has_location_uncertainty() && !handle.state.lock().await.calls.values().any(|c| { matches!( c.status, diff --git a/crates/cyber-server/src/runtime/binary_documents_tests.rs b/crates/cyber-server/src/runtime/binary_documents_tests.rs index 2be545b79..19bd785a2 100644 --- a/crates/cyber-server/src/runtime/binary_documents_tests.rs +++ b/crates/cyber-server/src/runtime/binary_documents_tests.rs @@ -225,11 +225,7 @@ async fn disposed_binary_request_retains_worker_ownership_until_actual_exit() { "timed-out Stop released the actual worker guard" ); let handle = runtime.inner.handle(&session).await.unwrap(); - assert!( - !handle - .location_uncertain - .load(std::sync::atomic::Ordering::SeqCst) - ); + assert!(!handle.has_location_uncertainty()); let shutdown = { let runtime = runtime.clone(); tokio::spawn(async move { runtime.shutdown().await }) @@ -372,11 +368,7 @@ async fn unwound_binary_worker_does_not_claim_file_success_or_poison_native_proc assert!(result.is_err()); assert_eq!(paths.available_permits(), 1); let handle = runtime.inner.handle(&session).await.unwrap(); - assert!( - !handle - .location_uncertain - .load(std::sync::atomic::Ordering::SeqCst) - ); + assert!(!handle.has_location_uncertainty()); assert!(runtime.inner.activities.lock().unwrap().is_empty()); runtime.shutdown().await; } diff --git a/crates/cyber-server/src/runtime/continuation.rs b/crates/cyber-server/src/runtime/continuation.rs index 3312cb986..0ef80f5d1 100644 --- a/crates/cyber-server/src/runtime/continuation.rs +++ b/crates/cyber-server/src/runtime/continuation.rs @@ -244,7 +244,7 @@ impl Runtime { state.ensure_worktree_ready()?; if state.child_continuation_unknown || handle.snapshot_active.load(Ordering::SeqCst) - || handle.location_uncertain.load(Ordering::SeqCst) + || handle.has_location_uncertainty() || state.calls.values().any(|call| { matches!( call.status, diff --git a/crates/cyber-server/src/runtime/location.rs b/crates/cyber-server/src/runtime/location.rs index 2e0de4f21..5ef210a98 100644 --- a/crates/cyber-server/src/runtime/location.rs +++ b/crates/cyber-server/src/runtime/location.rs @@ -68,7 +68,7 @@ impl Inner { if let Some(slot) = slot { lease = slot.empty(); } - let unsettled = handle.location_uncertain.load(Ordering::SeqCst) + let unsettled = handle.has_location_uncertainty() || handle.state.lock().await.calls.values().any(|call| { matches!( call.status, @@ -129,7 +129,7 @@ impl Inner { } }).await }).await; - if !handle.location_uncertain.load(Ordering::SeqCst) { + if !handle.has_location_uncertainty() { scope.settle()?; } result diff --git a/crates/cyber-server/src/runtime/location_uncertainty.rs b/crates/cyber-server/src/runtime/location_uncertainty.rs new file mode 100644 index 000000000..4a7580992 --- /dev/null +++ b/crates/cyber-server/src/runtime/location_uncertainty.rs @@ -0,0 +1,65 @@ +//! Opaque native outcomes and exact snapshot owners have separate recovery authority. +use super::{Handle, RuntimeError, activity}; +use cyber_store::Store; +use std::collections::HashSet; +use std::sync::PoisonError; +use std::sync::atomic::Ordering; + +impl Handle { + pub(super) fn has_location_uncertainty(&self) -> bool { + let snapshots = self + .snapshot_uncertain + .lock() + .unwrap_or_else(PoisonError::into_inner); + self.location_uncertain.load(Ordering::SeqCst) || !snapshots.is_empty() + } + + pub(super) fn mark_snapshot_uncertain(&self, activity: &str) { + self.snapshot_uncertain + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(activity.to_owned()); + } + + /// Call only at the writer after observing this exact owner's durable + /// acknowledgement. Opaque uncertainty and other owners are unaffected. + pub(super) fn acknowledge_snapshot_uncertainty(&self, activity: &str) { + self.snapshot_uncertain + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(activity); + } +} + +pub(super) fn load(store: &Store, session: &str) -> Result<(bool, HashSet), RuntimeError> { + let session = session.to_owned(); + let rows = store.read(move |db| { + let mut query = db.prepare("SELECT e.aggregate_id,e.data FROM event e WHERE e.type='native.activity.changed.1' AND json_extract(e.data,'$.session_id')=?1 AND json_extract(e.data,'$.status')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) LIMIT 4097")?; + Ok(query.query_map([session], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))?.collect::, _>>()?) + })?; + if rows.len() > 4096 { + return Err(RuntimeError::Conflict( + "Native Location uncertainty limit exceeded; retained evidence requires inspection" + .into(), + )); + } + let mut opaque = false; + let mut snapshots = HashSet::new(); + for (id, data) in rows { + let record: activity::Record = serde_json::from_str(&data) + .map_err(|error| RuntimeError::Corrupt(error.to_string()))?; + if !matches!(record.status.as_str(), "pending" | "unknown") { + return Err(RuntimeError::Corrupt( + "Invalid native activity status".into(), + )); + } + if record.snapshot_worker && record.binary_document.is_none() { + snapshots.insert(id); + } else if record.binary_document.is_none() || record.snapshot_worker { + // Document-specific workers retain their separate document recovery + // contract. Ambiguous mixed kinds never gain snapshot authority. + opaque = true; + } + } + Ok((opaque, snapshots)) +} diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index a924a3478..95df3b967 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -101,6 +101,7 @@ pub use subtree_reopen::SubtreeReopenReport; pub use subtree_stop::{SubtreeStopReport, SubtreeStopStatus}; mod interactive; mod location; +mod location_uncertainty; mod mcp; mod mcp_depth; pub mod messaging; @@ -745,7 +746,9 @@ pub(crate) struct Handle { pub title_requested: std::sync::atomic::AtomicBool, /// Set by a reject without feedback or a dismissed question: stop after the current tool group. pub halt: std::sync::atomic::AtomicBool, + /// Opaque process/checkout outcomes cannot be cleared by snapshot proof. pub location_uncertain: std::sync::atomic::AtomicBool, + snapshot_uncertain: StdMutex>, /// Fingerprints of calls since the last promoted input, for doom-loop detection. pub recent_calls: StdMutex>, /// Serialized auto reviews and consecutive blocks, reset for each new Drain. @@ -2587,17 +2590,15 @@ impl Inner { RuntimeError::Invalid("browser-managed Session capacity reached".into()) })?; } - let snapshot_source = id.to_owned(); - let snapshot_unknown = self.store.read(move |db| { - Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM event e WHERE e.type='native.activity.changed.1' AND json_extract(e.data,'$.session_id')=?1 AND json_extract(e.data,'$.snapshot_worker')=1 AND json_extract(e.data,'$.status')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq))", [snapshot_source], |row| row.get::<_, bool>(0))?) - })?; + let (opaque_unknown, snapshot_unknown) = location_uncertainty::load(&self.store, id)?; let handle = Arc::new(Handle { title_requested: std::sync::atomic::AtomicBool::new(!state.info.default_title), state: Mutex::new(state), snapshot_active: std::sync::atomic::AtomicBool::new(false), pending_compaction: StdMutex::default(), halt: std::sync::atomic::AtomicBool::new(false), - location_uncertain: std::sync::atomic::AtomicBool::new(snapshot_unknown), + location_uncertain: std::sync::atomic::AtomicBool::new(opaque_unknown), + snapshot_uncertain: StdMutex::new(snapshot_unknown), recent_calls: StdMutex::default(), auto_blocks: Mutex::new(0), stop_continuations: std::sync::atomic::AtomicUsize::new(0), diff --git a/crates/cyber-server/src/runtime/names.rs b/crates/cyber-server/src/runtime/names.rs index 7cce16623..a46ac2ab7 100644 --- a/crates/cyber-server/src/runtime/names.rs +++ b/crates/cyber-server/src/runtime/names.rs @@ -271,10 +271,7 @@ impl Runtime { )); } let handle = self.inner.handle(&owner.id).await?; - if handle - .location_uncertain - .load(std::sync::atomic::Ordering::SeqCst) - { + if handle.has_location_uncertainty() { return Err(RuntimeError::Invalid( "Location outcome unknown; recovery is required".into(), )); @@ -398,10 +395,7 @@ impl Runtime { )); } let handle = self.inner.handle(&owner.id).await?; - if handle - .location_uncertain - .load(std::sync::atomic::Ordering::SeqCst) - { + if handle.has_location_uncertainty() { return Err(RuntimeError::Invalid( "Location outcome unknown; recovery is required".into(), )); diff --git a/crates/cyber-server/src/runtime/rewind_preview.rs b/crates/cyber-server/src/runtime/rewind_preview.rs index 11df2eb0b..3ca2a29c8 100644 --- a/crates/cyber-server/src/runtime/rewind_preview.rs +++ b/crates/cyber-server/src/runtime/rewind_preview.rs @@ -553,13 +553,14 @@ impl Runtime { vec![revert_event(session_id, &revert, RevertPhase::Clear)] } }; + let owned_commit = completion.is_some(); let committed = if let Some(completed) = completion { completed.commit(&mut state, events) } else { self.inner.commit_locked(&mut state, events).map(drop) }; if let Err(error) = committed { - if preview.code.is_some() { + if preview.code.is_some() && !owned_commit { handle .location_uncertain .store(true, std::sync::atomic::Ordering::SeqCst); diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index ec8266fc1..02820f14a 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -239,7 +239,9 @@ impl SnapshotCompletion { self.native .guard .acknowledge(self.worker.scope.snapshot_origin()) - .map_err(RuntimeError::Conflict) + .map_err(|error| RuntimeError::Conflict(format!( + "Snapshot runtime acknowledgement committed; native journal acknowledgement requires reconciliation: {error}" + ))) } } diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index c76eb9368..d41b37e1d 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -62,8 +62,8 @@ fn restart(runtime: &Runtime, root: &std::path::Path) -> Runtime { #[tokio::test] async fn late_snapshot_disposal_preserves_durable_acknowledgement_and_unrelated_uncertainty() { - let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; - for unrelated_uncertainty in [false, true] { + for (unrelated_uncertainty, other_snapshot) in [(false, false), (true, false), (false, true)] { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; let owner = worker(&runtime, &session).await; let native_binding = binding(temp.path().canonicalize().unwrap()); let completed = owner @@ -99,16 +99,24 @@ async fn late_snapshot_disposal_preserves_durable_acknowledgement_and_unrelated_ handle .location_uncertain .store(unrelated_uncertainty, Ordering::SeqCst); + if other_snapshot { + handle.mark_snapshot_uncertain("op_other_snapshot"); + } drop(completed); assert!(notifications.try_recv().is_err()); assert_eq!(runtime.inner.store.aggregate_seq(&id).unwrap(), sequence); assert_eq!(latest(&runtime, &session), receipt); + assert_eq!( + handle.has_location_uncertainty(), + unrelated_uncertainty || other_snapshot + ); assert_eq!( handle.location_uncertain.load(Ordering::SeqCst), unrelated_uncertainty ); + assert!(!handle.snapshot_uncertain.lock().unwrap().contains(&id)); + runtime.shutdown().await; } - runtime.shutdown().await; } #[tokio::test] @@ -152,12 +160,90 @@ async fn late_snapshot_disposal_refuses_changed_native_ownership() { .handle(&session) .await .unwrap() - .location_uncertain - .load(Ordering::SeqCst) + .has_location_uncertainty() ); runtime.shutdown().await; } +#[tokio::test] +async fn reload_preserves_generic_uncertainty_separately_from_snapshot_owners() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + assert!( + owner + .run(move |work| { + work.begin(native_binding).unwrap(); + Err(RestoreError::Failed("interrupted snapshot".into())) + }) + .await + .is_err() + ); + let handle = runtime.inner.handle(&session).await.unwrap(); + assert!(handle.has_location_uncertainty()); + assert!(!handle.location_uncertain.load(Ordering::SeqCst)); + let snapshot = latest(&runtime, &session); + let activity_id = snapshot["snapshot_origin"]["activity_id"] + .as_str() + .unwrap() + .to_owned(); + assert!( + handle + .snapshot_uncertain + .lock() + .unwrap() + .contains(&activity_id) + ); + let mut generic = activity::Scope::reserve(&runtime.inner, &handle) + .await + .unwrap(); + generic.launch().unwrap(); + drop(generic); + assert!(handle.location_uncertain.load(Ordering::SeqCst)); + let reloaded = restart(&runtime, temp.path()); + let restored = reloaded.inner.handle(&session).await.unwrap(); + assert!(restored.location_uncertain.load(Ordering::SeqCst)); + assert!( + restored + .snapshot_uncertain + .lock() + .unwrap() + .contains(&activity_id) + ); + assert!( + reloaded + .admit( + &session, + Admission::text("must stay fenced", Delivery::Hold) + ) + .await + .is_err() + ); + // Simulate a separate writer's exact durable snapshot acknowledgement. Reload + // must retain the generic fence even when no snapshot activity remains unknown. + let mut receipt = snapshot; + receipt["status"] = "settled".into(); + runtime + .inner + .store + .append( + &activity_id, + cyber_store::Expected::Seq(2), + vec![NewEvent::new(activity::CHANGED, receipt)], + ) + .unwrap(); + let after_ack = restart(&runtime, temp.path()); + let restored = after_ack.inner.handle(&session).await.unwrap(); + assert!(restored.snapshot_uncertain.lock().unwrap().is_empty()); + assert!(restored.location_uncertain.load(Ordering::SeqCst)); + let state = restored.state.lock().await; + assert!(after_ack.snapshot_worker(&restored, &state).await.is_err()); + drop(state); + after_ack.shutdown().await; + reloaded.shutdown().await; + runtime.shutdown().await; +} + #[tokio::test] async fn ordinary_restore_commit_cannot_consume_a_recovery_receipt_or_acknowledge_it() { let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; @@ -332,7 +418,7 @@ async fn snapshot_completion_commits_session_and_exact_origin_before_native_ack( .unwrap(); assert_eq!(unrelated.events.last().unwrap().data["status"], "unknown"); assert!( - handle.location_uncertain.load(Ordering::SeqCst), + handle.has_location_uncertainty(), "snapshot receipt cleared another process's unknown outcome" ); assert!(!handle.snapshot_active.load(Ordering::SeqCst)); @@ -454,7 +540,7 @@ async fn snapshot_pre_effect_refusal_does_not_manufacture_an_unknown_native_effe assert!(record.get("snapshot_origin").is_none()); let handle = runtime.inner.handle(&session).await.unwrap(); assert!(!handle.snapshot_active.load(Ordering::SeqCst)); - assert!(!handle.location_uncertain.load(Ordering::SeqCst)); + assert!(!handle.has_location_uncertainty()); runtime.shutdown().await; } @@ -546,8 +632,7 @@ async fn rejected_runtime_commit_keeps_origin_unknown_and_never_acknowledges_the .handle(&session) .await .unwrap() - .location_uncertain - .load(Ordering::SeqCst) + .has_location_uncertainty() ); restarted.shutdown().await; } diff --git a/crates/cyber-server/src/runtime/subtree_reopen.rs b/crates/cyber-server/src/runtime/subtree_reopen.rs index 3ac78111b..5e7e99690 100644 --- a/crates/cyber-server/src/runtime/subtree_reopen.rs +++ b/crates/cyber-server/src/runtime/subtree_reopen.rs @@ -9,7 +9,6 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::{BTreeSet, HashSet}; -use std::sync::atomic::Ordering; use std::time::Duration; pub(super) const REOPENED: &str = "session.admission.reopened.1"; @@ -134,7 +133,7 @@ impl Runtime { let mut proofs = Vec::with_capacity(locations.len()); for info in locations { let handle = self.inner.handle(&info.id).await?; - if handle.location_uncertain.load(Ordering::SeqCst) { + if handle.has_location_uncertainty() { return Err(RuntimeError::Conflict( "Native Location requires reviewed recovery".into(), )); diff --git a/crates/cyber-server/src/runtime/subtree_stop.rs b/crates/cyber-server/src/runtime/subtree_stop.rs index a00eb550d..2f0de0592 100644 --- a/crates/cyber-server/src/runtime/subtree_stop.rs +++ b/crates/cyber-server/src/runtime/subtree_stop.rs @@ -273,9 +273,7 @@ impl Runtime { for id in ids { let handle = self.inner.handle(id).await?; let state = handle.state.lock().await.clone(); - if handle - .location_uncertain - .load(std::sync::atomic::Ordering::SeqCst) + if handle.has_location_uncertainty() || state.child_continuation_unknown || state.child_worktree_setup_pending || state.open_step.is_some() diff --git a/crates/cyber-server/src/runtime/worktree_events.rs b/crates/cyber-server/src/runtime/worktree_events.rs index 29a2ca82d..9470f174f 100644 --- a/crates/cyber-server/src/runtime/worktree_events.rs +++ b/crates/cyber-server/src/runtime/worktree_events.rs @@ -192,7 +192,7 @@ impl Runtime { /// Location and the call runs alone. [`Self::enter_worktree`] checks again. pub async fn check_enterable(&self, id: &str, call_id: &str) -> Result<(), RuntimeError> { let handle = self.inner.handle(id).await?; - if handle.location_uncertain.load(Ordering::SeqCst) { + if handle.has_location_uncertainty() { return Err(RuntimeError::Invalid( "Location outcome unknown; recovery is required".into(), )); @@ -334,7 +334,7 @@ impl Runtime { ) -> Result, RuntimeError> { let _admission = self.inner.open().await?; let handle = self.inner.handle(id).await?; - if handle.location_uncertain.load(Ordering::SeqCst) { + if handle.has_location_uncertainty() { return Err(RuntimeError::Invalid( "Location outcome unknown; recovery is required".into(), )); @@ -388,7 +388,7 @@ impl Runtime { ) -> Result<(), RuntimeError> { let _admission = self.inner.open().await?; let handle = self.inner.handle(id).await?; - if handle.location_uncertain.load(Ordering::SeqCst) { + if handle.has_location_uncertainty() { return Err(RuntimeError::Invalid( "Location outcome unknown; recovery is required".into(), )); diff --git a/crates/cyber-server/tests/rewind_preview.rs b/crates/cyber-server/tests/rewind_preview.rs index 9ae9dc0e7..688f92ac0 100644 --- a/crates/cyber-server/tests/rewind_preview.rs +++ b/crates/cyber-server/tests/rewind_preview.rs @@ -12,6 +12,7 @@ use support::*; #[derive(Default)] struct Images { fail: AtomicBool, + owned: AtomicBool, supported: AtomicBool, restored: AtomicUsize, gate_diff: AtomicBool, @@ -19,6 +20,38 @@ struct Images { release: tokio::sync::Notify, } impl Snapshots for Images { + fn owned_restore_available(&self, _: &str) -> bool { + self.owned.load(Ordering::SeqCst) + } + fn restore_owned( + &self, + request: SnapshotRestoreRequest, + worker: SnapshotWorker, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + self.restored.fetch_add(1, Ordering::SeqCst); + let fail_ack = self.fail.load(Ordering::SeqCst); + worker + .run(move |work| { + let origin = work + .begin(SnapshotBinding { + directory: std::fs::canonicalize(request.directory).unwrap(), + root_identity: "instrumented_root".into(), + namespace_identity: "instrumented_namespace".into(), + journal_id: "restore_instrumented".into(), + journal_identity: "instrumented_journal".into(), + lock_identity: Some("instrumented_lock".into()), + }) + .map_err(RestoreError::Failed)?; + work.effect(|| Ok(())).map_err(RestoreError::Failed)?; + Ok(SnapshotNativeResult::new( + vec![], + Box::new(JournalAck { origin, fail_ack }), + )) + }) + .await + }) + } fn track(&self, _: &str) -> BoxFuture<'_, Result, String>> { Box::pin(async { Ok(Some(Snapshot { @@ -81,6 +114,27 @@ impl Snapshots for Images { }) } } +struct JournalAck { + origin: SnapshotOrigin, + fail_ack: bool, +} +impl SnapshotNativeGuard for JournalAck { + fn verify(&self, origin: Option<&SnapshotOrigin>) -> Result<(), String> { + if origin == Some(&self.origin) { + Ok(()) + } else { + Err("foreign origin".into()) + } + } + fn acknowledge(&mut self, origin: Option<&SnapshotOrigin>) -> Result<(), String> { + self.verify(origin)?; + if self.fail_ack { + Err("injected journal flush failure".into()) + } else { + Ok(()) + } + } +} fn runtime_with_images(h: &Harness, images: Arc) -> Runtime { Runtime::new(RuntimeOptions { store: h.store.clone(), @@ -130,6 +184,59 @@ fn harness() -> Harness { }) } +#[tokio::test] +async fn journal_ack_failure_preserves_committed_rewind_without_fabricating_unknown_native_effects() +{ + let h = harness(); + let images = Arc::new(Images::default()); + images.supported.store(true, Ordering::SeqCst); + images.owned.store(true, Ordering::SeqCst); + images.fail.store(true, Ordering::SeqCst); + let runtime = runtime_with_images(&h, images.clone()); + let (id, selection) = seed(&h, &runtime).await; + let before = runtime.state(&id).await.unwrap(); + let preview = runtime.rewind_preview(&id, &selection).await.unwrap(); + let confirmation = RewindConfirmation { + selection, + confirmation: preview.confirmation, + }; + let error = runtime + .rewind_confirm(&id, &confirmation) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("runtime acknowledgement committed"), + "{error}" + ); + let after = runtime.state(&id).await.unwrap(); + assert!(after.last_seq > before.last_seq); + assert!(after.entries.is_empty()); + let source = id.clone(); + let status: String = h.store.read(move |db| { + Ok(db.query_row("SELECT json_extract(data,'$.status') FROM event WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 ORDER BY rowid DESC LIMIT 1", [source], |row| row.get(0))?) + }).unwrap(); + assert_eq!(status, "settled"); + assert!(runtime.rewind_confirm(&id, &confirmation).await.is_err()); + assert_eq!(images.restored.load(Ordering::SeqCst), 1); + runtime + .admit( + &id, + Admission::text("held after durable native acknowledgement", Delivery::Hold), + ) + .await + .unwrap(); + runtime.shutdown().await; + let restarted = runtime_with_images(&h, images.clone()); + restarted + .admit(&id, Admission::text("held after reload", Delivery::Hold)) + .await + .unwrap(); + assert_eq!(images.restored.load(Ordering::SeqCst), 1); + restarted.shutdown().await; +} + #[tokio::test] async fn failed_native_ack_keeps_activity_unknown_without_moving_the_branch() { let h = harness(); diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index e07ee43b8..76a0d9a33 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -645,3 +645,57 @@ its in-memory state with recovery acknowledgement, including acknowledgement tha follows a successful unknown-disposal append. Git, runtime request ownership, reconciliation, cleanup and native/client acceptance remain open. No Linux/Windows, full-workspace or real recovery operator acceptance was run for this safeguard. + +### Snapshot uncertainty provenance and committed journal-ack failure + +The Handle now retains exact snapshot activity IDs separately from opaque native +process/checkout uncertainty. All existing Location uncertainty checks combine +both reasons. Reload reconstructs both from durable latest activity records, +bounded to 4096 unresolved owners; it no longer folds snapshots into an opaque +boolean or drops generic native fences. Document-specific workers keep their +separate document recovery contract, and ambiguous mixed worker kinds are opaque. +No legacy record gains snapshot authority from reload. + +Snapshot disposal publishes its exact in-memory reason before enqueueing its +writer decision. If that writer observes the already-settled exact owner, it +removes only that reason. There is no caller-side post-commit snapshot flag write +that can re-fence an acknowledgement after the writer has consumed it. The opaque +flag and every other snapshot ID survive. Combined reads hold the snapshot mutex +while reading the opaque flag so removing a snapshot reason cannot hide an opaque +reason published before that removal. The eventual consuming recovery transaction +still needs its exact native proof and post-commit reconciliation. + +Owned rewind now delegates uncertainty to the actual worker instead of marking an +opaque outcome for every commit error. If the Session/native writer pair commits +and the later journal acknowledgement fails, the error explicitly reports the +committed runtime acknowledgement and need for journal reconciliation. The native +activity stays settled, the committed branch change remains visible, and repeating +the stale rewind confirmation cannot replay the restore. This does not implement +the missing journal reconciliation operation or authorize cleanup. + +The restart regression combines an interrupted snapshot and generic process, then +simulates an independent exact snapshot acknowledgement and verifies that reload +still refuses new snapshot work because of the generic fence. The late-disposal +regression covers no unrelated reason, an opaque reason and another snapshot ID +in independent fixtures. A new scripted-provider server integration exercises the +owned rewind confirm path with an injected journal-ack failure, checks the committed +branch/native state, stale-confirm refusal, held input admission before/after +reload, and exactly one restore invocation. Filesystem identities and the failed +journal flush are instrumented; this is not native recovery/reconciliation proof. + +Local macOS checks passed: 397 server unit tests (one ignored subprocess fixture, +invoked by its parent Location identity tests), four idle-cancellation, thirteen +Location activity and five rewind-preview integration cases. Strict OpenSpec +passed all 117 items; server all-target Clippy with warnings denied and +formatting/diff checks passed. No external model provider, +new public schema or SDK contract is involved. Linux/Windows, full-workspace, +real CLI/TUI recovery, native resolution crash cuts and lost-writer-reply journal +reconciliation remain unperformed for this increment. M2–M8 and both issues remain +open. + +Substantive code/security review covered all combined reads, opaque writers, +reload classification, exact-ID removal at the writer, failure/lifetime ordering, +the owned versus legacy commit paths and shared binary worker behavior. It +corrected the potential post-commit flag race and combined-read ordering. The +broader recovery authority, native consuming transaction and reconciliation gates +remain required before operator apply or cleanup is admitted. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 88f5d532b..6e553d462 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -64,6 +64,13 @@ activity. Acquiring the exact bound store/checkout locks plus validating that or is proof about this worker only. It cannot acknowledge a shell, child process, other document, another runtime's live worker or an unrelated unknown activity. +Runtime Location uncertainty distinguishes exact snapshot activity IDs from opaque +process/checkout outcomes. Reload preserves both kinds independently. Observing an +exact durable snapshot acknowledgement can remove only that snapshot reason; it +cannot clear the opaque reason or another snapshot owner. A committed runtime +acknowledgement followed by failed journal acknowledgement requires reconciliation, +without reclassifying the already-committed native effect as unknown. + Resolution is available through a dedicated owner recovery path while ordinary admission is fenced. It does not use a blanket admission bypass. It requires idle Session/Location scope, no active origin worker, exact captured authority and diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 2b5038896..cb1e4603d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -55,9 +55,12 @@ Late original-scope disposal now checks durable identity and status within the writer transaction after native store-lock release. It preserves an already acknowledged activity and its complete receipt; changed ownership/sequence refuses the append. This does not implement recovery acknowledgement. Before enabling M5, -preserve opaque/generic Location uncertainty; -the current shared `location_uncertain` boolean is not proof that every reason -belongs to the resolved snapshot. Reconciliation must distinguish a durable writer +preserve opaque/generic Location uncertainty. Snapshot uncertainty is now tracked +by exact activity ID separately from the opaque flag; reload reconstructs generic +fences and snapshot owners independently. A late-scope no-op clears only its exact +snapshot reason after observing the matching durable acknowledgement at the writer. +The consuming recovery commit still needs to apply this distinction. Reconciliation +must distinguish a durable writer acknowledgement with missing journal acknowledgement from an uncommitted filesystem outcome, and must not revalidate or replay later owner edits as the old request. Unprepared reservation/origin/intent cuts still require exact resource ownership From f30639245b1ee346c61e6506b98021f18271bd67 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 19:38:44 -0400 Subject: [PATCH 07/87] Add owned non-Git recovery admission and paired settlement --- crates/cyber-cli/tests/branches_cli.rs | 186 +++++++ crates/cyber-server/src/http/openapi.rs | 5 + crates/cyber-server/src/http/sessions.rs | 37 ++ crates/cyber-server/src/runtime/activity.rs | 229 ++++++++- .../src/runtime/admission_authority.rs | 5 + .../cyber-server/src/runtime/auto_override.rs | 2 +- crates/cyber-server/src/runtime/hooks.rs | 2 +- crates/cyber-server/src/runtime/mod.rs | 9 +- .../src/runtime/snapshot_recovery.rs | 269 ++++++++++- .../runtime/snapshot_recovery_authority.rs | 95 ++++ .../src/runtime/snapshot_recovery_record.rs | 180 +++++++ .../src/runtime/snapshot_recovery_types.rs | 29 ++ .../cyber-server/src/runtime/snapshot_work.rs | 106 +++- .../src/runtime/snapshot_work_tests.rs | 453 +++++++++++++++++- crates/cyber-snapshot/src/copy/mod.rs | 8 + crates/cyber-snapshot/src/copy/resolution.rs | 145 +++++- .../src/copy/resolution_tests.rs | 131 +++++ crates/cyber-snapshot/src/copy/restore.rs | 42 +- crates/cyber-snapshot/src/lib.rs | 10 + .../implementation.md | 95 ++++ .../managed-recovery.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 61 ++- sdk/openapi.json | 233 +++++++++ sdk/typescript/src/generated/operations.ts | 42 ++ sdk/typescript/src/generated/types.ts | 36 ++ sdk/typescript/test/rewind.test.ts | 15 + 26 files changed, 2351 insertions(+), 81 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_recovery_authority.rs create mode 100644 crates/cyber-server/src/runtime/snapshot_recovery_record.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 3cfbbe02f..e5a7594bc 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -784,6 +784,192 @@ fn recovery_review_binds_the_session_discloses_acknowledgement_and_starts_no_inf assert_eq!(model.requests().len(), requests); } +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn managed_keep_and_rollback_after_real_partial_restore_kill_preserve_owner_files_and_never_replay() +{ + managed_native_recovery_after_partial_kill("keep"); + managed_native_recovery_after_partial_kill("rollback"); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn managed_native_recovery_after_partial_kill(action: &str) { + let model = scripted_model::ScriptedModel::start(reply); + let env = Env::new(&model); + let repo = env.root.join("repo"); + std::fs::remove_dir_all(repo.join(".git")).unwrap(); + // Enough separately flushed exchanges to observe an actual partial source + // namespace before killing the real server; no restore fault hook is used. + for index in 0..1280 { + std::fs::write(repo.join(format!("bulk_{index:04}")), "before").unwrap(); + } + let id = env.exec(None, "before snapshot"); + let first = env.user_ids(&id)[0].clone(); + for index in 0..1280 { + std::fs::write(repo.join(format!("bulk_{index:04}")), "after").unwrap(); + } + env.exec(Some(&id), "after snapshot"); + let messages = kill_partial_restore(&env, &id, &first); + let requests = model.requests().len(); + std::fs::write(repo.join("unrelated-owner"), "owner data").unwrap(); + if action == "keep" { + std::fs::write(repo.join("bulk_1279"), "later owner image").unwrap(); + } + let catalog = json(&env.cyber(&["api", "GET", &format!("/sessions/{id}/rewind/recovery")])); + let journals = catalog["data"].as_array().unwrap(); + assert_eq!(journals.len(), 1); + assert_eq!(journals[0]["filesystem_complete"], false); + let journal = journals[0]["id"].as_str().unwrap(); + let review = json(&env.cyber(&[ + "api", + "POST", + &format!("/sessions/{id}/rewind/recovery/review"), + "-d", + &json!({"journal_id":journal,"action":action}).to_string(), + ]))["data"] + .clone(); + assert!( + review["conflicts"].as_array().unwrap().is_empty(), + "{review}" + ); + let request_id = format!("real-kill-{action}"); + let request = json!({"journal_id":journal,"action":action,"confirmation":review["confirmation"],"request_id":request_id}); + let confirm_path = format!("/sessions/{id}/rewind/recovery/confirm"); + let result = + json(&env.cyber(&["api", "POST", &confirm_path, "-d", &request.to_string()]))["data"] + .clone(); + assert_eq!(result["state"], "acknowledged"); + assert_eq!(result["journal_acknowledged"], true); + assert_eq!(env.file("unrelated-owner").as_deref(), Some("owner data")); + let expected = if action == "rollback" { + "after" + } else { + "before" + }; + assert_eq!(env.file("bulk_0000").as_deref(), Some(expected)); + assert_eq!(env.messages(&id), messages); + assert_eq!( + model.requests().len(), + requests, + "recovery started inference" + ); + assert_native_settlement_link(&env, journal, &result); + assert_recovery_lookup_does_not_replay(&env, &id, &request_id, &request, &result, &messages); + assert_eq!(model.requests().len(), requests); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn kill_partial_restore(env: &Env, id: &str, first: &str) -> Vec { + let selection = reviewed_selection(env, first, "both", "commit"); + let preview = reviewed_preview(env, id, &selection); + let mut body = selection; + body["confirmation"] = preview["confirmation"].clone(); + let body = body.to_string(); + let path = format!("/sessions/{id}/rewind/confirm"); + let mut pending = env + .command(&["api", "POST", &path, "-d", &body]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(45); + while env.file("bulk_0000").as_deref() != Some("before") { + assert!( + pending.try_wait().unwrap().is_none(), + "restore exited before a source exchange" + ); + assert!( + std::time::Instant::now() < deadline, + "source exchange was never observed" + ); + std::thread::sleep(Duration::from_millis(1)); + } + assert_eq!( + env.file("bulk_1279").as_deref(), + Some("after"), + "no partial restore window was observed" + ); + kill_registered_service(env); + assert!(!pending.wait_with_output().unwrap().status.success()); + let messages = env.messages(id); // Starts a fresh server over the durable evidence. + assert_eq!(env.file("bulk_0000").as_deref(), Some("before")); + assert_eq!( + env.file("bulk_1279").as_deref(), + Some("after"), + "kill did not retain a partial namespace" + ); + assert_eq!( + env.user_ids(id).len(), + 2, + "failed restore moved the transcript" + ); + messages +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_native_settlement_link(env: &Env, journal: &str, result: &Value) { + let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let old: Value = serde_json::from_slice( + &std::fs::read(namespace.join("journals").join(journal).join("intent.json")).unwrap(), + ) + .unwrap(); + assert_eq!(old["completed"], false); + assert_eq!(old["acknowledged"], true); + assert_eq!(old["settled_by"], result["receipt"]["resolution_id"]); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_recovery_lookup_does_not_replay( + env: &Env, + id: &str, + request_id: &str, + request: &Value, + result: &Value, + messages: &[Value], +) { + let repo = env.root.join("repo"); + let confirm_path = format!("/sessions/{id}/rewind/recovery/confirm"); + std::fs::write(repo.join("bulk_0000"), "new owner edit after recovery").unwrap(); + let retry = + json(&env.cyber(&["api", "POST", &confirm_path, "-d", &request.to_string()]))["data"] + .clone(); + assert_eq!(retry["state"], "acknowledged"); + assert!( + retry["journal_acknowledged"].is_null(), + "lookup invented native acknowledgement observation" + ); + assert_eq!( + env.file("bulk_0000").as_deref(), + Some("new owner edit after recovery") + ); + let mut changed = request.clone(); + changed["confirmation"] = "b".repeat(64).into(); + assert!( + !env.cyber(&["api", "POST", &confirm_path, "-d", &changed.to_string()]) + .status + .success(), + "changed request content reused an existing owner" + ); + kill_registered_service(env); + let lookup = json(&env.cyber(&[ + "api", + "GET", + &format!("/sessions/{id}/rewind/recovery/requests/{request_id}"), + ]))["data"] + .clone(); + assert_eq!(lookup["receipt"], result["receipt"]); + assert_eq!( + env.file("bulk_0000").as_deref(), + Some("new owner edit after recovery") + ); + assert_eq!(env.messages(id).as_slice(), messages); +} + #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn reviewed_non_git_stage_and_clear_keep_exact_acknowledged_worker_receipts() { diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index 134de48d9..3082f725f 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -763,6 +763,11 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { op("post", "/sessions/{sessionID}/rewind/recovery/review", "v1.session.snapshotRecoveryReview", "Review retained snapshot effects without restoring, acknowledging native activity or starting inference") .body::(g) .ok::>(g), + op("post", "/sessions/{sessionID}/rewind/recovery/confirm", "v1.session.snapshotRecoveryConfirm", "Resolve fingerprint-confirmed native snapshot effects with an exact durable request; repeated IDs are lookup-only") + .body::(g) + .ok::>(g), + op("get", "/sessions/{sessionID}/rewind/recovery/requests/{requestID}", "v1.session.snapshotRecoveryLookup", "Look up a durable recovery request without native access, acknowledgement reconciliation, replay or inference") + .ok::>(g), op("post", "/sessions/{sessionID}/rewind/preview", "v1.session.rewindPreview", "Review a rewind without changing files or the active conversation") .body::(g) .ok::>(g), diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index fef94d88b..e4c10225d 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -350,6 +350,14 @@ pub fn routes() -> Router { &format!("{s}/rewind/recovery/review"), post(snapshot_recovery_review), ) + .route( + &format!("{s}/rewind/recovery/confirm"), + post(snapshot_recovery_confirm), + ) + .route( + &format!("{s}/rewind/recovery/requests/{{request_id}}"), + get(snapshot_recovery_lookup), + ) .route(&format!("{s}/rewind/confirm"), post(rewind_confirm)) .route(&format!("{s}/revert/stage"), post(stage)) .route(&format!("{s}/revert/clear"), post(clear)) @@ -1213,6 +1221,35 @@ async fn snapshot_recovery_review( .await?, })) } +async fn snapshot_recovery_confirm( + State(state): State, + Path(id): Path, + Json(confirmation): Json, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_recovery_confirm(&id, &confirmation) + .await?, + })) +} +async fn snapshot_recovery_lookup( + State(state): State, + Path((id, request_id)): Path<(String, String)>, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_recovery_lookup(&id, &request_id) + .await? + .ok_or_else(|| { + ApiError::not_found( + "RequestNotFoundError", + "No retained snapshot recovery request", + ) + })?, + })) +} async fn rewind_preview( State(state): State, Path(id): Path, diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index ba7e77d28..c6ea82efe 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -30,6 +30,10 @@ pub(super) struct Record { #[serde(default, skip_serializing_if = "Option::is_none")] pub snapshot_origin: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_request: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_receipt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_bindings: Option>, } pub(super) struct Scope { @@ -37,18 +41,39 @@ pub(super) struct Scope { inner: Weak, store: Arc, handle: Arc, - authority: AdmissionAuthority, + authority: Authority, record: Record, pub control: Arc, launched: bool, settled: bool, } +enum Authority { + Ordinary(AdmissionAuthority), + Recovery(super::snapshot_recovery_authority::RecoveryAuthority), +} impl Scope { pub fn id(&self) -> &str { &self.id } - pub fn authority(&self) -> AdmissionAuthority { - self.authority.clone() + pub fn authority(&self) -> Result { + match &self.authority { + Authority::Ordinary(authority) => Ok(authority.clone()), + Authority::Recovery(_) => Err(RuntimeError::Conflict( + "Snapshot recovery cannot lend ordinary launch authority".into(), + )), + } + } + pub(super) fn verify_authority(&self) -> Result<(), RuntimeError> { + let runtime = Runtime { + inner: self.inner.upgrade().ok_or(RuntimeError::ShuttingDown)?, + }; + if self.control.stop.is_cancelled() { + return Err(RuntimeError::Conflict("Native worker was cancelled".into())); + } + match &self.authority { + Authority::Ordinary(authority) => authority.verify(&runtime, &self.control.source), + Authority::Recovery(authority) => authority.verify(&runtime), + } } pub async fn reserve(inner: &Inner, handle: &Handle) -> Result { Self::reserve_kind(inner, handle, None).await @@ -75,6 +100,66 @@ impl Scope { ) -> Result { Self::reserve_owner(inner, source, None, true).await } + pub(super) fn reserve_recovery( + runtime: &Runtime, + handle: Arc, + id: String, + request: super::snapshot_recovery_types::RecoveryRequest, + authority: super::snapshot_recovery_authority::RecoveryAuthority, + ) -> Result { + let source = request.original.session_id.clone(); + let record = Record { + session_id: source.clone(), + status: "pending".into(), + phase: "reserved".into(), + binary_document: None, + snapshot_worker: true, + snapshot_origin: None, + snapshot_request: Some(request.clone()), + snapshot_receipt: None, + admission_bindings: Some(authority.bindings().to_vec()), + }; + let captured = authority.clone(); + let closed = runtime.inner.closed.clone(); + let event = change(&record); + runtime + .inner + .store + .append_checked(&id, Expected::Seq(-1), move |tx| { + let (_, original) = super::snapshot_recovery_record::attest(tx, &request.original)?; + if closed.is_cancelled() || !captured.current(tx)? || original.status == "settled" { + return Err(cyber_store::StoreError::InvalidAppend( + "Snapshot recovery reservation authority changed".into(), + )); + } + Ok((vec![event], ())) + })?; + let control = Arc::new(Control { + source, + binary_document: None, + snapshot_worker: true, + admission_sessions: super::admission_authority::binding_sessions(authority.bindings()), + stop: runtime.inner.closed.child_token(), + done: CancellationToken::new(), + }); + runtime + .inner + .activities + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(id.clone(), control.clone()); + Ok(Self { + id, + inner: Arc::downgrade(&runtime.inner), + store: runtime.inner.store.clone(), + handle, + authority: Authority::Recovery(authority), + record, + control, + launched: false, + settled: false, + }) + } async fn reserve_owner( inner: &Inner, source: String, @@ -94,6 +179,8 @@ impl Scope { binary_document, snapshot_worker, snapshot_origin: None, + snapshot_request: None, + snapshot_receipt: None, admission_bindings: Some(authority.bindings.clone()), }; inner @@ -117,7 +204,7 @@ impl Scope { inner: Arc::downgrade(&runtime.inner), store: inner.store.clone(), handle: owned_handle, - authority, + authority: Authority::Ordinary(authority), record, control, launched: false, @@ -125,10 +212,7 @@ impl Scope { }) } pub fn launch(&mut self) -> Result<(), RuntimeError> { - let runtime = Runtime { - inner: self.inner.upgrade().ok_or(RuntimeError::ShuttingDown)?, - }; - self.authority.verify(&runtime, &self.control.source)?; + self.verify_authority()?; if self.control.stop.is_cancelled() { return Err(RuntimeError::Invalid( "Idle operation cancelled before launch".into(), @@ -173,7 +257,8 @@ impl Scope { let runtime = Runtime { inner: self.inner.upgrade().ok_or(RuntimeError::ShuttingDown)?, }; - self.authority.verify(&runtime, &self.control.source)?; + let authority = self.authority()?; + authority.verify(&runtime, &self.control.source)?; if state.last_seq != sequence || !self.record.snapshot_worker || state.info.id != self.control.source @@ -187,7 +272,7 @@ impl Scope { settled.status = "settled".into(); settled.phase = "snapshot_restore_committed".into(); settled.admission_bindings = None; - let bindings = self.authority.bindings.clone(); + let bindings = authority.bindings.clone(); let acknowledgement = change(&settled); let owners = [ (state.info.id.clone(), state.last_seq), @@ -210,6 +295,11 @@ impl Scope { Ok(()) } pub fn settle(&mut self) -> Result<(), RuntimeError> { + if matches!(self.authority, Authority::Recovery(_)) { + return Err(RuntimeError::Conflict( + "Recovery requires its paired native acknowledgement".into(), + )); + } self.record.status = "settled".into(); self.record.admission_bindings = None; self.store @@ -217,11 +307,112 @@ impl Scope { self.settled = true; Ok(()) } + pub(super) fn commit_recovery( + &mut self, + state: &super::SessionState, + sequence: i64, + receipt: &super::SnapshotRecoveryReceipt, + ) -> Result<(), RuntimeError> { + self.verify_authority()?; + let Authority::Recovery(authority) = &self.authority else { + return Err(RuntimeError::Conflict( + "Ordinary activity cannot acknowledge recovery".into(), + )); + }; + let request = self + .record + .snapshot_request + .as_ref() + .ok_or_else(|| RuntimeError::Conflict("Missing recovery request owner".into()))?; + let origin = self + .snapshot_origin() + .ok_or_else(|| RuntimeError::Conflict("Missing resolution origin".into()))?; + let original = &request.original; + if state.info.id != self.control.source + || state.last_seq != sequence + || receipt.original != *original + || !receipt.filesystem_complete + || receipt.request_id != request.confirmation.request_id + || receipt.journal_id != request.confirmation.journal_id + || receipt.confirmation != request.confirmation.confirmation + || receipt.action != request.confirmation.action + || receipt.resolution_id != origin.binding.journal_id + || origin.binding.journal_id == original.binding.journal_id + || origin.binding.directory != original.binding.directory + || origin.binding.root_identity != original.binding.root_identity + || origin.binding.namespace_identity != original.binding.namespace_identity + || origin.binding.lock_identity.is_none() + || origin.binding.lock_identity != original.binding.lock_identity + { + return Err(RuntimeError::Conflict( + "Resolution receipt differs from its exact request and native origin".into(), + )); + } + let original = original.clone(); + let origin = origin.clone(); + let original_sequence = self + .store + .aggregate_seq(&original.activity_id)? + .ok_or_else(|| RuntimeError::Conflict("Original native owner disappeared".into()))?; + let mut settled = self.record.clone(); + settled.status = "settled".into(); + settled.phase = "snapshot_recovery_committed".into(); + settled.admission_bindings = None; + settled.snapshot_receipt = Some(receipt.clone()); + let resolution_event = change(&settled); + let authority = authority.clone(); + let receipt = receipt.clone(); + let expected_request = request.clone(); + let stop = self.control.stop.clone(); + let owners = [ + (original.activity_id.clone(), original_sequence), + (self.id.clone(), 1), + ]; + self.store.append_pair_checked(owners, move |tx| { + let (_, mut original_record) = super::snapshot_recovery_record::attest(tx, &original)?; + let (_, resolution_record) = super::snapshot_recovery_record::attest(tx, &origin)?; + let current: i64 = tx.query_row( + "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", + [&original.session_id], + |row| row.get(0), + )?; + let created: String = tx.query_row( + "SELECT id FROM event WHERE aggregate_id=?1 ORDER BY seq LIMIT 1", + [&original.session_id], + |row| row.get(0), + )?; + if stop.is_cancelled() + || !authority.current(tx)? + || current != sequence + || created != original.session_created_id + || original_record.status == "settled" + || resolution_record.status != "pending" + || resolution_record.snapshot_request.as_ref() != Some(&expected_request) + { + return Err(cyber_store::StoreError::InvalidAppend( + "Recovery consuming acknowledgement authority changed".into(), + )); + } + original_record.status = "settled".into(); + original_record.phase = "snapshot_recovery_committed".into(); + original_record.admission_bindings = None; + original_record.snapshot_receipt = Some(receipt); + Ok(([vec![change(&original_record)], vec![resolution_event]], ())) + })?; + self.handle + .acknowledge_snapshot_uncertainty(&request.original.activity_id); + self.record = settled; + self.settled = true; + Ok(()) + } pub async fn run(&self, future: impl std::future::Future) -> T { + let Authority::Ordinary(authority) = &self.authority else { + return CANCEL.scope(self.control.stop.clone(), future).await; + }; CANCEL .scope( self.control.stop.clone(), - self.authority + authority .clone() .with_cancellation(self.control.done.clone()) .scope(self.control.stop.clone(), future), @@ -236,7 +427,7 @@ impl Scope { // Publish before enqueueing. Even an unavailable writer retains the exact // owner; a later writer acknowledgement can clear it without racing a // post-commit caller-side flag write. - if self.launched && original.snapshot_worker { + if (self.launched || original.snapshot_request.is_some()) && original.snapshot_worker { handle.mark_snapshot_uncertain(&id); } let (_, uncertain) = self.store.append_checked(&self.id, Expected::Any, move |tx| { @@ -257,6 +448,7 @@ impl Scope { || current.binary_document != original.binary_document || current.snapshot_worker != original.snapshot_worker || current.snapshot_origin.as_ref() != origin + || current.snapshot_request != original.snapshot_request { return Err(cyber_store::StoreError::InvalidAppend("Native activity ownership changed before disposal".into())); } @@ -287,12 +479,13 @@ impl Drop for Scope { // possible file effects remain in the separate document intent. let mut disposed = self.record.clone(); disposed.snapshot_origin = self.snapshot_origin().cloned(); - disposed.status = if self.launched && !filesystem_only { - "unknown" - } else { - "settled" - } - .into(); + disposed.status = + if (self.launched || self.record.snapshot_request.is_some()) && !filesystem_only { + "unknown" + } else { + "settled" + } + .into(); if filesystem_only { disposed.phase = "filesystem_worker_disposed_query_document".into(); } diff --git a/crates/cyber-server/src/runtime/admission_authority.rs b/crates/cyber-server/src/runtime/admission_authority.rs index e7ee91322..0817deaaf 100644 --- a/crates/cyber-server/src/runtime/admission_authority.rs +++ b/crates/cyber-server/src/runtime/admission_authority.rs @@ -558,6 +558,10 @@ pub(super) fn project(tx: &Transaction<'_>, event: &StoredEvent) -> Result<(), S if event.kind == super::hooks::STARTED && event.data["synthetic"] == true { return Ok(()); } + if super::snapshot_recovery_record::project_pending(tx, event)? { + return Ok(()); + } + super::snapshot_recovery_record::check_ordinary_admission(tx, event)?; check_closed_admission(tx, event)?; let source = match event.kind.as_str() { super::events::CREATED => event.data["info"]["parent_id"].as_str().map(str::to_string), @@ -659,6 +663,7 @@ fn check_closed_admission(tx: &Transaction<'_>, event: &StoredEvent) -> Result<( _ => return Ok(()), }; for target in targets { + super::snapshot_recovery_record::check_session_admission(tx, target, &event.aggregate_id)?; let chain = snapshot_chain(tx, target) .map_err(|e| e.to_string())? .ok_or("Admission ancestry has a cycle or a missing ancestor")?; diff --git a/crates/cyber-server/src/runtime/auto_override.rs b/crates/cyber-server/src/runtime/auto_override.rs index 293b2f775..bdfb5fa4c 100644 --- a/crates/cyber-server/src/runtime/auto_override.rs +++ b/crates/cyber-server/src/runtime/auto_override.rs @@ -460,7 +460,7 @@ impl Inner { call_id: replay.id.clone(), tool: replay.name.clone(), input, - authority: scope.authority(), + authority: scope.authority()?, cancel: scope.control.stop.clone(), done: invocation_done.clone(), }); diff --git a/crates/cyber-server/src/runtime/hooks.rs b/crates/cyber-server/src/runtime/hooks.rs index c355d7e5e..176b08449 100644 --- a/crates/cyber-server/src/runtime/hooks.rs +++ b/crates/cyber-server/src/runtime/hooks.rs @@ -345,7 +345,7 @@ impl Runtime { } let native_observation = self.validate_native_file_hook(event)?; let activity = super::activity::Scope::reserve(&self.inner, &handle).await?; - let authority = activity.authority(); + let authority = activity.authority()?; authority.verify(self, &event.identity().session_id)?; let mut record = admission_record(event, definition, log_io); let observation = super::file_observation_hooks::prepare(event, definition, &mut record)?; diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 95df3b967..acf322071 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -107,6 +107,8 @@ mod mcp_depth; pub mod messaging; pub mod peers; pub mod presence; +mod snapshot_recovery_authority; +mod snapshot_recovery_record; pub use mcp_depth::RunMarker; mod memory; pub use mcp::{ @@ -132,8 +134,8 @@ pub use rewind_preview::{ }; pub use snapshot_recovery_types::{ SnapshotRecoveryAction, SnapshotRecoveryConfirmation, SnapshotRecoveryImage, - SnapshotRecoveryPath, SnapshotRecoveryReceipt, SnapshotRecoveryReview, - SnapshotRecoverySelection, + SnapshotRecoveryPath, SnapshotRecoveryReceipt, SnapshotRecoveryResult, SnapshotRecoveryReview, + SnapshotRecoverySelection, SnapshotRecoveryState, }; pub use snapshot_work::{ SnapshotBinding, SnapshotCompletion, SnapshotNativeGuard, SnapshotNativeResult, SnapshotOrigin, @@ -403,7 +405,8 @@ impl From for RuntimeError { match error { StoreError::Projector { reason, .. } if reason == admission_authority::STALE - || reason == admission_authority::CLOSED => + || reason == admission_authority::CLOSED + || reason == snapshot_recovery_record::FENCED => { Self::Conflict(reason) } diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index 04de345e5..023f8d221 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -3,6 +3,155 @@ use super::{Runtime, RuntimeError, SnapshotRecoveryReview, SnapshotRecoverySelec use rusqlite::OptionalExtension; impl Runtime { + /// A repeated request is always lookup-only, even after a partial outcome or + /// later owner edits. Reconciliation is a separate operation. + pub async fn snapshot_recovery_confirm( + &self, + session: &str, + confirmation: &super::SnapshotRecoveryConfirmation, + ) -> Result { + validate_request(confirmation)?; + if let Some(existing) = self + .snapshot_recovery_lookup(session, &confirmation.request_id) + .await? + { + if existing.request != *confirmation { + return Err(RuntimeError::Conflict( + "Recovery request ID was reused with different content".into(), + )); + } + return Ok(existing); + } + let review = self + .snapshot_recovery_review( + session, + &SnapshotRecoverySelection { + journal_id: confirmation.journal_id.clone(), + action: confirmation.action, + }, + ) + .await?; + if review.confirmation.as_deref() != Some(&confirmation.confirmation) { + return Err(RuntimeError::Conflict( + "Recovery confirmation is stale or unavailable".into(), + )); + } + let original = review + .origin + .ok_or_else(|| RuntimeError::Conflict("Recovery lacks exact native origin".into()))?; + let handle = self.inner.handle(session).await?; + let state = handle.state.lock().await; + let directory = state.info.directory.clone(); + let worker = self + .snapshot_recovery_worker( + &handle, + &state, + super::snapshot_recovery_types::RecoveryRequest { + confirmation: confirmation.clone(), + original, + authority: None, + }, + ) + .await?; + drop(state); + let completed = self + .inner + .options + .snapshots + .resolve_recovery(directory, confirmation.clone(), worker) + .await + .map_err(|error| { + RuntimeError::Conflict(format!( + "Recovery retains its request and native evidence: {error:?}" + )) + })?; + let state = handle.state.lock().await; + completed.commit_recovery(&state)?; + drop(state); + let mut result = self + .snapshot_recovery_lookup(session, &confirmation.request_id) + .await? + .ok_or_else(|| { + RuntimeError::Corrupt("Committed recovery request disappeared".into()) + })?; + result.journal_acknowledged = Some(true); + Ok(result) + } + + /// Durable request lookup never opens a native namespace or starts inference. + pub async fn snapshot_recovery_lookup( + &self, + session: &str, + request_id: &str, + ) -> Result, RuntimeError> { + validate_request_id(request_id)?; + self.session_info(session).await?; + let created = self + .inner + .store + .read_events(session, -1, 1)? + .events + .first() + .ok_or_else(|| RuntimeError::Conflict("Recovery Session creation is missing".into()))? + .id + .clone(); + let id = request_owner(&created, request_id); + let owner = id.clone(); + let data = self.inner.store.read(move |db| { + Ok(db + .query_row( + "SELECT type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [owner], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?) + })?; + let Some((kind, data)) = data else { + return Ok(None); + }; + let record: activity::Record = serde_json::from_str(&data) + .map_err(|error| RuntimeError::Corrupt(error.to_string()))?; + let request = record.snapshot_request.ok_or_else(|| { + RuntimeError::Conflict("Request ID belongs to another native operation".into()) + })?; + if kind != activity::CHANGED + || !record.snapshot_worker + || record.binary_document.is_some() + || record.session_id != session + || request.original.session_id != session + || request.original.session_created_id != created + || request.confirmation.request_id != request_id + || record + .snapshot_origin + .as_ref() + .is_some_and(|origin| origin.activity_id != id) + { + return Err(RuntimeError::Conflict( + "Recovery request owner differs from the current Session".into(), + )); + } + let state = match (record.status.as_str(), &record.snapshot_receipt) { + ("pending", None) => super::SnapshotRecoveryState::Pending, + ("unknown", None) | ("settled", None) => super::SnapshotRecoveryState::Unknown, + ("settled", Some(receipt)) if receipt.request_id == request_id => { + super::SnapshotRecoveryState::Acknowledged + } + ("settled", Some(_)) => super::SnapshotRecoveryState::ResolvedByAnotherRequest, + _ => { + return Err(RuntimeError::Corrupt( + "Invalid recovery request settlement".into(), + )); + } + }; + Ok(Some(super::SnapshotRecoveryResult { + request: request.confirmation, + original: request.original, + resolution: record.snapshot_origin, + state, + receipt: record.snapshot_receipt, + journal_acknowledged: None, + })) + } pub async fn snapshot_recovery_review( &self, session: &str, @@ -60,6 +209,18 @@ impl Runtime { review: &mut SnapshotRecoveryReview, ) -> Result<(), RuntimeError> { let origin = review.origin.as_ref().expect("caller checked origin"); + if origin + .binding + .lock_identity + .as_deref() + .is_none_or(str::is_empty) + { + review.confirmation = None; + review + .conflicts + .push("Legacy snapshot origin has no exact native lock proof".into()); + return Ok(()); + } let id = origin.activity_id.clone(); let rows = self.inner.store.read(move |db| { let latest = db.query_row("SELECT data FROM event WHERE aggregate_id=?1 AND type='native.activity.changed.1' ORDER BY seq DESC LIMIT 1", [&id], |row| row.get::<_,String>(0)).optional()?; @@ -114,6 +275,47 @@ impl Runtime { } } +fn validate_request_id(id: &str) -> Result<(), RuntimeError> { + if id.is_empty() || id.len() > 256 || id.contains('\0') { + return Err(RuntimeError::Invalid( + "Recovery request ID must contain 1–256 bytes without NUL".into(), + )); + } + Ok(()) +} +pub(super) fn validate_request( + request: &super::SnapshotRecoveryConfirmation, +) -> Result<(), RuntimeError> { + validate_request_id(&request.request_id)?; + if request.confirmation.len() != 64 + || !request + .confirmation + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || !request.journal_id.starts_with("restore_") + || request.journal_id.len() > 256 + || !request + .journal_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err(RuntimeError::Invalid( + "Invalid recovery journal or fingerprint".into(), + )); + } + Ok(()) +} +pub(super) fn request_owner(created: &str, request: &str) -> String { + use sha2::{Digest, Sha256}; + let mut digest = Sha256::new(); + digest.update(b"snapshot-recovery-request-v1"); + digest.update((created.len() as u64).to_be_bytes()); + digest.update(created.as_bytes()); + digest.update((request.len() as u64).to_be_bytes()); + digest.update(request.as_bytes()); + format!("op_snapshot_recovery_{:x}", digest.finalize()) +} + #[cfg(all(test, any(target_os = "macos", target_os = "linux")))] mod tests { use super::*; @@ -192,14 +394,31 @@ mod tests { .unwrap(), sequence ); - // Use an independently owned typed scope to exercise acknowledged state. - let mut settled = activity::Scope::reserve_snapshot(&runtime.inner, session.clone()) - .await + // Model an independently committed acknowledgement of this exact owner; + // ordinary admission must not create another scope through its fence. + let mut acknowledged_record = runtime + .inner + .store + .read_events(&bound.activity_id, -1, 10) + .unwrap() + .events + .last() + .unwrap() + .data + .clone(); + acknowledged_record["status"] = "settled".into(); + runtime + .inner + .store + .append( + &bound.activity_id, + cyber_store::Expected::Seq(sequence.unwrap()), + vec![cyber_store::NewEvent::new( + activity::CHANGED, + acknowledged_record, + )], + ) .unwrap(); - let bound = origin(&runtime, &session, settled.id()); - settled.launch_snapshot(bound.clone()).unwrap(); - settled.settle().unwrap(); - drop(settled); let mut acknowledged = review(bound); runtime .validate_recovery_activity(&mut acknowledged) @@ -226,6 +445,7 @@ mod tests { runtime.inner.store.aggregate_seq(scope.id()).unwrap(), before ); + scope.settle().unwrap(); drop(scope); let mut typed = activity::Scope::reserve_snapshot(&runtime.inner, session.clone()) .await @@ -241,6 +461,41 @@ mod tests { drop(typed); runtime.shutdown().await; } + #[tokio::test] + async fn legacy_origin_without_exact_lock_proof_stays_inspectable_and_fenced() { + let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let mut scope = activity::Scope::reserve_snapshot(&runtime.inner, session.clone()) + .await + .unwrap(); + let mut bound = origin(&runtime, &session, scope.id()); + bound.binding.lock_identity = None; + scope.launch_snapshot(bound.clone()).unwrap(); + drop(scope); + let sequence = runtime + .inner + .store + .aggregate_seq(&bound.activity_id) + .unwrap(); + let mut legacy = review(bound.clone()); + runtime.validate_recovery_activity(&mut legacy).unwrap(); + assert!(legacy.confirmation.is_none()); + assert!( + legacy + .conflicts + .iter() + .any(|conflict| conflict.contains("exact native lock proof")) + ); + assert_eq!( + runtime + .inner + .store + .aggregate_seq(&bound.activity_id) + .unwrap(), + sequence + ); + runtime.shutdown().await; + } + #[tokio::test] async fn missing_native_evidence_disables_confirmation_without_creating_activity() { let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_authority.rs b/crates/cyber-server/src/runtime/snapshot_recovery_authority.rs new file mode 100644 index 000000000..84a1881a7 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_recovery_authority.rs @@ -0,0 +1,95 @@ +//! Snapshot-only authority may preserve a closed boundary, never reopen or inherit it. +use super::{Runtime, RuntimeError, admission_authority}; +use cyber_store::StoreError; +use rusqlite::Connection; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RecoveryAuthority { + bindings: Vec, + solve_sequences: Vec, + closed_scopes: Vec>, +} + +impl RecoveryAuthority { + pub fn capture(runtime: &Runtime, source: &str) -> Result { + let source = source.to_owned(); + runtime + .inner + .store + .read(move |db| { + let tx = db.unchecked_transaction()?; + let bindings = + admission_authority::snapshot_chain(&tx, &source)?.ok_or_else(|| { + StoreError::InvalidAppend("Recovery ancestry is unavailable".into()) + })?; + let solve_sequences = solve_sequences(&tx, &bindings)?; + if bindings.len() > 4096 { + return Err(StoreError::InvalidAppend( + "Recovery ancestry exceeds limit".into(), + )); + } + let closed_scopes = closed_scopes(&tx, &bindings)?; + tx.commit()?; + Ok(Self { + bindings, + solve_sequences, + closed_scopes, + }) + }) + .map_err(Into::into) + } + pub fn bindings(&self) -> &[admission_authority::Binding] { + &self.bindings + } + pub fn current(&self, db: &Connection) -> Result { + let Some(source) = self.bindings.first() else { + return Ok(false); + }; + Ok( + admission_authority::snapshot_chain(db, &source.id)?.as_ref() == Some(&self.bindings) + && solve_sequences(db, &self.bindings)? == self.solve_sequences + && closed_scopes(db, &self.bindings)? == self.closed_scopes, + ) + } + pub fn verify(&self, runtime: &Runtime) -> Result<(), RuntimeError> { + let captured = self.clone(); + let current = runtime.inner.store.read(move |db| { + let tx = db.unchecked_transaction()?; + let current = captured.current(&tx)?; + tx.commit()?; + Ok(current) + })?; + if !current { + return Err(RuntimeError::Conflict( + "Snapshot recovery authority changed".into(), + )); + } + Ok(()) + } +} +fn closed_scopes( + db: &Connection, + bindings: &[admission_authority::Binding], +) -> Result>, StoreError> { + bindings + .iter() + .map(|binding| super::subtree_reopen::active_scope(db, &binding.id)) + .collect() +} +fn solve_sequences( + db: &Connection, + bindings: &[admission_authority::Binding], +) -> Result, StoreError> { + bindings + .iter() + .map(|binding| { + Ok(db.query_row( + "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", + [format!("solve_fence_{}", binding.id)], + |row| row.get(0), + )?) + }) + .collect() +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_record.rs b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs new file mode 100644 index 000000000..5cf388965 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs @@ -0,0 +1,180 @@ +//! Exact durable native owners; a Session-wide unknown outcome is never an acknowledgement target. +use super::{SnapshotOrigin, activity}; +use cyber_store::StoreError; +use rusqlite::{Connection, OptionalExtension}; + +pub(super) const FENCED: &str = "Native snapshot ownership fences ordinary native admission"; + +pub(super) fn check_session_admission( + db: &Connection, + source: &str, + exclude: &str, +) -> Result<(), String> { + let fenced: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM event e WHERE e.type='native.activity.changed.1' AND e.aggregate_id<>?1 AND json_extract(e.data,'$.session_id')=?2 AND json_extract(e.data,'$.snapshot_worker')=1 AND json_extract(e.data,'$.status')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq))", [exclude, source], |row| row.get(0)).map_err(|error| error.to_string())?; + if fenced { + return Err(FENCED.into()); + } + Ok(()) +} + +pub(super) fn check_ordinary_admission( + db: &Connection, + event: &cyber_store::StoredEvent, +) -> Result<(), String> { + if event.kind != activity::CHANGED || event.data["status"] != "pending" { + return Ok(()); + } + let source = event.data["session_id"] + .as_str() + .ok_or("Missing native activity source")?; + let snapshot = event.data["snapshot_worker"] == true; + let fenced: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM event e WHERE e.type='native.activity.changed.1' AND e.aggregate_id<>?1 AND json_extract(e.data,'$.session_id')=?2 AND json_extract(e.data,'$.status')<>'settled' AND (?3 OR json_extract(e.data,'$.snapshot_worker')=1) AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq))", rusqlite::params![event.aggregate_id, source, snapshot], |row| row.get(0)).map_err(|error| error.to_string())?; + if fenced { + return Err(FENCED.into()); + } + Ok(()) +} + +/// The only closed-boundary admission exception is an exact, reserved snapshot +/// resolution. This cannot authorize an ordinary hook/process/child launch. +pub(super) fn project_pending( + db: &Connection, + event: &cyber_store::StoredEvent, +) -> Result { + if event.kind != activity::CHANGED + || event.data["status"] != "pending" + || event.data["snapshot_request"].is_null() + { + return Ok(false); + } + let record: activity::Record = + serde_json::from_value(event.data.clone()).map_err(|error| error.to_string())?; + let request = record + .snapshot_request + .as_ref() + .ok_or("Missing recovery request")?; + super::snapshot_recovery::validate_request(&request.confirmation) + .map_err(|error| error.to_string())?; + let authority = request + .authority + .as_ref() + .ok_or("Missing captured recovery authority")?; + let (_, original) = attest(db, &request.original).map_err(|error| error.to_string())?; + let created: String = db + .query_row( + "SELECT id FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq LIMIT 1", + [&record.session_id, super::events::CREATED], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if !record.snapshot_worker + || record.binary_document.is_some() + || record.snapshot_receipt.is_some() + || original.status == "settled" + || request + .original + .binding + .lock_identity + .as_deref() + .is_none_or(str::is_empty) + || record.session_id != request.original.session_id + || created != request.original.session_created_id + || event.aggregate_id == request.original.activity_id + || event.aggregate_id + != super::snapshot_recovery::request_owner(&created, &request.confirmation.request_id) + || record.admission_bindings.as_deref() != Some(authority.bindings()) + || authority + .bindings() + .first() + .map(|binding| binding.id.as_str()) + != Some(record.session_id.as_str()) + || !authority.current(db).map_err(|error| error.to_string())? + { + return Err("Invalid snapshot-only recovery reservation authority".into()); + } + match event.seq { + 0 if record.phase == "reserved" && record.snapshot_origin.is_none() => {} + 1 if record.phase == "launching" => { + let data: String = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=0 AND type=?2", + [&event.aggregate_id, activity::CHANGED], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + let reserved: activity::Record = + serde_json::from_str(&data).map_err(|error| error.to_string())?; + let origin = record + .snapshot_origin + .as_ref() + .ok_or("Missing resolution origin")?; + if reserved.snapshot_request != record.snapshot_request + || reserved.status != "pending" + || origin.activity_id != event.aggregate_id + || origin.session_id != record.session_id + || origin.session_created_id != created + || origin.worktree_id != request.original.worktree_id + || origin.binding.directory != request.original.binding.directory + || origin.binding.root_identity != request.original.binding.root_identity + || origin.binding.namespace_identity != request.original.binding.namespace_identity + || origin.binding.lock_identity.is_none() + || origin.binding.lock_identity != request.original.binding.lock_identity + || origin.binding.journal_id == request.original.binding.journal_id + { + return Err("Resolution launch differs from its exact request reservation".into()); + } + } + _ => return Err("Invalid snapshot recovery launch sequence".into()), + } + Ok(true) +} + +pub(super) fn attest( + db: &Connection, + origin: &SnapshotOrigin, +) -> Result<(i64, activity::Record), StoreError> { + let launched = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=1 AND type=?2", + [&origin.activity_id, activity::CHANGED], + |row| row.get::<_, String>(0), + ) + .optional()?; + let latest = db + .query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&origin.activity_id], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .optional()?; + let Some((launched, (seq, kind, latest))) = launched.zip(latest) else { + return Err(StoreError::InvalidAppend( + "Exact snapshot origin is unavailable".into(), + )); + }; + let launched: activity::Record = serde_json::from_str(&launched) + .map_err(|error| StoreError::InvalidAppend(error.to_string()))?; + let latest: activity::Record = serde_json::from_str(&latest) + .map_err(|error| StoreError::InvalidAppend(error.to_string()))?; + if kind != activity::CHANGED + || [&launched, &latest].iter().any(|record| { + !record.snapshot_worker + || record.binary_document.is_some() + || record.session_id != origin.session_id + || record.snapshot_origin.as_ref() != Some(origin) + }) + || launched.snapshot_request != latest.snapshot_request + || !matches!(latest.status.as_str(), "pending" | "unknown" | "settled") + { + return Err(StoreError::InvalidAppend( + "Native activity differs from the exact snapshot origin".into(), + )); + } + Ok((seq, latest)) +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs index 695c74364..09f70f7cd 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs @@ -76,3 +76,32 @@ pub struct SnapshotRecoveryReceipt { pub original: super::SnapshotOrigin, pub filesystem_complete: bool, } + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(super) struct RecoveryRequest { + pub confirmation: SnapshotRecoveryConfirmation, + pub original: super::SnapshotOrigin, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authority: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SnapshotRecoveryState { + Pending, + Unknown, + Acknowledged, + ResolvedByAnotherRequest, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SnapshotRecoveryResult { + pub request: SnapshotRecoveryConfirmation, + pub original: super::SnapshotOrigin, + pub resolution: Option, + pub state: SnapshotRecoveryState, + pub receipt: Option, + /// Lookup does not inspect native files. None means journal acknowledgement + /// has not been observed by this response, even if runtime settlement is durable. + pub journal_acknowledged: Option, +} diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 02820f14a..812132824 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -117,8 +117,7 @@ impl SnapshotWork<'_> { } self.owner .scope - .authority() - .verify(&self.owner.runtime, &self.owner.scope.control.source) + .verify_authority() .map_err(|error| error.to_string())?; let source = self.owner.scope.control.source.clone(); let sequence = self.owner.sequence; @@ -219,6 +218,28 @@ impl SnapshotCompletion { pub fn recovery_receipt(&self) -> Option<&super::SnapshotRecoveryReceipt> { self.native.recovery.as_ref() } + pub(crate) fn commit_recovery( + mut self, + state: &SessionState, + ) -> Result { + let receipt = self + .native + .recovery + .as_ref() + .ok_or_else(|| RuntimeError::Conflict("Missing recovery filesystem receipt".into()))? + .clone(); + self.native + .guard + .verify(self.worker.scope.snapshot_origin()) + .map_err(RuntimeError::Conflict)?; + self.worker + .scope + .commit_recovery(state, self.worker.sequence, &receipt)?; + self.native.guard.acknowledge(self.worker.scope.snapshot_origin()).map_err(|error| RuntimeError::Conflict(format!( + "Recovery runtime acknowledgement committed; native journal acknowledgement requires reconciliation: {error}" + )))?; + Ok(receipt) + } pub(crate) fn commit( mut self, state: &mut SessionState, @@ -246,6 +267,87 @@ impl SnapshotCompletion { } impl Runtime { + pub(super) async fn snapshot_recovery_worker( + &self, + handle: &Arc, + state: &SessionState, + mut request: super::snapshot_recovery_types::RecoveryRequest, + ) -> Result { + let lifecycle = self.memory_mutation_lease().await?; + self.inner.ensure_idle(&state.info.id)?; + self.ensure_no_background(&state.info.id)?; + if self + .inner + .activities + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .any(|control| control.source == state.info.id) + { + return Err(RuntimeError::Busy(state.info.id.clone())); + } + state.ensure_worktree_ready()?; + if request.original.session_id != state.info.id + || request.original.worktree_id != state.info.worktree_id + { + return Err(RuntimeError::Conflict( + "Recovery Session/checkout changed".into(), + )); + } + handle + .snapshot_active + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .map_err(|_| RuntimeError::Busy(state.info.id.clone()))?; + let admission = SnapshotAdmission(handle.clone()); + let authority = + super::snapshot_recovery_authority::RecoveryAuthority::capture(self, &state.info.id)?; + request.authority = Some(authority.clone()); + let id = super::snapshot_recovery::request_owner( + &request.original.session_created_id, + &request.confirmation.request_id, + ); + let scope = activity::Scope::reserve_recovery( + self, + handle.clone(), + id, + request.clone(), + authority, + )?; + let lease = self + .inner + .claim_location(&state.info, false, scope.control.stop.clone()) + .await?; + let directory = std::fs::canonicalize(&state.info.directory) + .map_err(|error| RuntimeError::Invalid(error.to_string()))?; + let created_id = self + .inner + .store + .read_events(&state.info.id, -1, 1)? + .events + .first() + .ok_or_else(|| RuntimeError::Conflict("Recovery Session origin missing".into()))? + .id + .clone(); + if directory != request.original.binding.directory + || created_id != request.original.session_created_id + || lease.worktree_id != request.original.worktree_id + { + return Err(RuntimeError::Conflict( + "Recovery native Location/creation binding changed".into(), + )); + } + Ok(SnapshotWorker { + scope, + admission, + sequence: state.last_seq, + runtime: self.clone(), + directory, + created_id, + worktree_id: state.info.worktree_id.clone(), + _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, + _lifecycle: lifecycle, + }) + } pub(crate) async fn snapshot_worker( &self, handle: &Arc, diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index d41b37e1d..e28537cec 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -43,9 +43,26 @@ fn latest(runtime: &Runtime, session: &str) -> serde_json::Value { Ok(serde_json::from_str(&data).unwrap()) }).unwrap() } + +fn legacy_generic_unknown(runtime: &Runtime, handle: &Handle, session: &str) -> String { + // Historical opaque evidence can coexist with a snapshot fence. This does + // not launch new work through an already-fenced ordinary admission path. + let id = cyber_core::ids::new_id("op"); + runtime.inner.store.append(&id, cyber_store::Expected::Seq(-1), vec![NewEvent::new(activity::CHANGED, + serde_json::json!({"session_id":session,"status":"unknown","phase":"legacy_process_unknown"}))]).unwrap(); + handle.location_uncertain.store(true, Ordering::SeqCst); + id +} fn restart(runtime: &Runtime, root: &std::path::Path) -> Runtime { + runtime_with_store(runtime, runtime.inner.store.clone(), root) +} +fn runtime_with_store( + runtime: &Runtime, + store: Arc, + root: &std::path::Path, +) -> Runtime { Runtime::new(RuntimeOptions { - store: runtime.inner.store.clone(), + store, resolver: runtime.inner.resolver.clone(), tools: runtime.inner.tools.clone(), global_config_dir: root.join("config"), @@ -60,6 +77,397 @@ fn restart(runtime: &Runtime, root: &std::path::Path) -> Runtime { }) } +#[tokio::test] +async fn recovery_pair_lost_commit_reply_is_lookup_only_after_reopening_the_real_store() { + fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("injected recovery pair reply loss".into()), + )) + } + let (temp, base, _, _, _) = super::super::binary_documents_tests::fixture().await; + let path = temp.path().join("recovery-pair.db"); + let store = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(path.clone()), + Runtime::registry(), + )) + .unwrap(); + let runtime = runtime_with_store(&base, Arc::new(store), temp.path()); + let session = runtime + .create_session(super::super::CreateSession { + directory: temp.path().to_str().unwrap().into(), + model: "test/main".into(), + ..Default::default() + }) + .await + .unwrap() + .id; + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let request = recovery_request(original.clone()); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + drop(state); + let count = Arc::new(AtomicUsize::new(0)); + let completed = finish_recovery(owner, request.clone(), count.clone()).await; + runtime + .inner + .store + .transaction(|tx| { + tx.wal_hook(Some(lost_reply)); + Ok(()) + }) + .unwrap(); + let notifications = runtime.inner.store.subscribe(); + let state = handle.state.lock().await; + assert!(completed.commit_recovery(&state).is_err()); + drop(state); + assert_eq!(count.load(Ordering::SeqCst), 0); + assert!( + notifications.try_recv().is_err(), + "unknown commit reply published a notification" + ); + let original_record = runtime + .inner + .store + .read_events(&original.activity_id, -1, 10) + .unwrap(); + assert_eq!( + original_record.events.last().unwrap().data["status"], + "settled" + ); + let lookup = runtime + .snapshot_recovery_confirm(&session, &request.confirmation) + .await + .unwrap(); + assert_eq!( + lookup.state, + super::super::SnapshotRecoveryState::Acknowledged + ); + assert_eq!(lookup.journal_acknowledged, None); + let receipt = lookup.receipt.unwrap(); + runtime.shutdown().await; + drop(handle); + drop(runtime); + let reopened = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(path), + Runtime::registry(), + )) + .unwrap(); + let reloaded = runtime_with_store(&base, Arc::new(reopened), temp.path()); + let lookup = reloaded + .snapshot_recovery_confirm(&session, &request.confirmation) + .await + .unwrap(); + assert_eq!(lookup.receipt, Some(receipt)); + assert_eq!( + lookup.state, + super::super::SnapshotRecoveryState::Acknowledged + ); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "lookup replayed native work" + ); + reloaded.shutdown().await; + base.shutdown().await; +} + +async fn failed_snapshot(runtime: &Runtime, session: &str, directory: PathBuf) -> SnapshotOrigin { + let owner = worker(runtime, session).await; + assert!( + owner + .run(move |work| { + work.begin(binding(directory)).unwrap(); + Err(RestoreError::Failed("interrupted native restore".into())) + }) + .await + .is_err() + ); + serde_json::from_value(latest(runtime, session)["snapshot_origin"].clone()).unwrap() +} + +fn recovery_request( + original: SnapshotOrigin, +) -> super::super::snapshot_recovery_types::RecoveryRequest { + super::super::snapshot_recovery_types::RecoveryRequest { + confirmation: super::super::SnapshotRecoveryConfirmation { + journal_id: original.binding.journal_id.clone(), + action: super::super::SnapshotRecoveryAction::Keep, + confirmation: "a".repeat(64), + request_id: "owned-recovery-request".into(), + }, + original, + authority: None, + } +} + +async fn finish_recovery( + owner: SnapshotWorker, + request: super::super::snapshot_recovery_types::RecoveryRequest, + count: Arc, +) -> SnapshotCompletion { + owner + .run(move |work| { + let mut native_binding = request.original.binding.clone(); + native_binding.journal_id = "restore_resolution_owned".into(); + native_binding.journal_identity = "resolution_journal_identity".into(); + let origin = work.begin(native_binding).unwrap(); + work.effect(|| Ok(())).unwrap(); + let mut native = SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin.clone()), + acknowledgements: count, + }), + ); + native.recovery = Some(super::super::SnapshotRecoveryReceipt { + journal_id: request.confirmation.journal_id, + resolution_id: origin.binding.journal_id, + request_id: request.confirmation.request_id, + action: request.confirmation.action, + confirmation: request.confirmation.confirmation, + original: request.original, + filesystem_complete: true, + }); + Ok(native) + }) + .await + .unwrap() +} + +#[tokio::test] +async fn writer_snapshot_fence_rejects_input_even_when_the_callers_precheck_is_stale() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let admitted = runtime + .admit(&session, Admission::text("held seed", Delivery::Hold)) + .await + .unwrap(); + let source = session.clone(); + let mut payload: serde_json::Value = runtime.inner.store.read(move |db| { + let data: String = db.query_row("SELECT data FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq DESC LIMIT 1", [&source, events::ADMITTED], |row| row.get(0))?; + Ok(serde_json::from_str(&data).unwrap()) + }).unwrap(); + payload["message_id"] = "msg_stale_admission".into(); + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let before = runtime.inner.store.aggregate_seq(&session).unwrap(); + let failed = runtime.inner.store.append( + &session, + cyber_store::Expected::Seq(before.unwrap()), + vec![NewEvent::new(events::ADMITTED, payload)], + ); + assert!( + matches!(failed, Err(cyber_store::StoreError::Projector { reason, .. }) if reason == super::super::snapshot_recovery_record::FENCED) + ); + assert_eq!(runtime.inner.store.aggregate_seq(&session).unwrap(), before); + let state = runtime.state(&session).await.unwrap(); + assert!(state.input(&admitted.message_id).is_some()); + assert!(state.input("msg_stale_admission").is_none()); + assert_eq!( + runtime + .inner + .store + .aggregate_seq(&original.activity_id) + .unwrap(), + Some(2) + ); + runtime.shutdown().await; +} + +#[tokio::test] +async fn ordinary_snapshot_reservation_cannot_race_a_pending_generic_native_owner() { + let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut generic = activity::Scope::reserve(&runtime.inner, &handle) + .await + .unwrap(); + let source = session.clone(); + let events_before: i64 = runtime.inner.store.read(move |db| Ok(db.query_row("SELECT count(*) FROM event WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1", [source], |row| row.get(0))?)).unwrap(); + let state = handle.state.lock().await; + assert!(runtime.snapshot_worker(&handle, &state).await.is_err()); + drop(state); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); + let source = session.clone(); + let events_after: i64 = runtime.inner.store.read(move |db| Ok(db.query_row("SELECT count(*) FROM event WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1", [source], |row| row.get(0))?)).unwrap(); + assert_eq!( + events_after, events_before, + "failed admission published native ownership" + ); + generic.launch().unwrap(); + generic.settle().unwrap(); + drop(generic); + runtime.shutdown().await; +} + +#[tokio::test] +async fn recovery_preserves_closed_admission_and_acknowledges_only_its_exact_native_pair() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let handle = runtime.inner.handle(&session).await.unwrap(); + legacy_generic_unknown(&runtime, &handle, &session); + let boundary = runtime.close_subtree_admissions(&session).await.unwrap(); + assert!(runtime.capture_child_admission(&session).is_err()); + let request = recovery_request(original.clone()); + let state = handle.state.lock().await; + let sequence = state.last_seq; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + assert!(owner.scope.authority().is_err()); + drop(state); + let reserved = runtime + .snapshot_recovery_lookup(&session, &request.confirmation.request_id) + .await + .unwrap() + .unwrap(); + assert_eq!(reserved.state, super::super::SnapshotRecoveryState::Pending); + assert!(reserved.resolution.is_none() && reserved.receipt.is_none()); + let count = Arc::new(AtomicUsize::new(0)); + let completed = finish_recovery(owner, request.clone(), count.clone()).await; + let state = handle.state.lock().await; + completed.commit_recovery(&state).unwrap(); + assert_eq!(state.last_seq, sequence, "recovery moved the transcript"); + drop(state); + assert_eq!(count.load(Ordering::SeqCst), 1); + assert!( + handle.location_uncertain.load(Ordering::SeqCst), + "generic unknown outcome was cleared" + ); + assert!( + !handle + .snapshot_uncertain + .lock() + .unwrap() + .contains(&original.activity_id) + ); + assert!(runtime.capture_child_admission(&session).is_err()); + let source = session.clone(); + assert_eq!( + runtime + .inner + .store + .read(move |db| super::super::subtree_reopen::active_scope(db, &source)) + .unwrap(), + Some(boundary) + ); + let lookup = runtime + .snapshot_recovery_confirm(&session, &request.confirmation) + .await + .unwrap(); + assert_eq!( + lookup.state, + super::super::SnapshotRecoveryState::Acknowledged + ); + assert!(lookup.receipt.is_some()); + assert_eq!(lookup.journal_acknowledged, None); + assert_eq!( + count.load(Ordering::SeqCst), + 1, + "retry replayed native acknowledgement" + ); + let mut foreign = request.confirmation; + foreign.journal_id = "restore_foreign".into(); + assert!( + runtime + .snapshot_recovery_confirm(&session, &foreign) + .await + .is_err() + ); + runtime.shutdown().await; +} + +#[tokio::test] +async fn a_new_stop_boundary_revokes_recovery_before_native_origin_or_effects() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let request = recovery_request(original); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + drop(state); + runtime.fence_subtree_admissions(&session).await.unwrap(); + let native_binding = binding(temp.path().canonicalize().unwrap()); + assert!( + owner + .run(move |work| { + assert!(work.authorize().is_err()); + assert!(work.begin(native_binding).is_err()); + assert!( + work.effect::<()>(|| panic!("revoked recovery changed source files")) + .is_err() + ); + Err(RestoreError::Failed("recovery revoked".into())) + }) + .await + .is_err() + ); + let lookup = runtime + .snapshot_recovery_confirm(&session, &request.confirmation) + .await + .unwrap(); + assert_eq!(lookup.state, super::super::SnapshotRecoveryState::Unknown); + assert!(lookup.resolution.is_none() && lookup.receipt.is_none()); + assert!(handle.has_location_uncertainty()); + runtime.shutdown().await; +} + +#[tokio::test] +async fn changed_session_rejects_recovery_pair_and_retry_only_looks_up_unknown_request() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let request = recovery_request(original.clone()); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + drop(state); + let count = Arc::new(AtomicUsize::new(0)); + let completed = finish_recovery(owner, request.clone(), count.clone()).await; + let mut state = handle.state.lock().await; + runtime + .inner + .commit_locked( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "changed revision".into(), + usage: None, + cost: None, + model: None, + }, + )], + ) + .unwrap(); + assert!(completed.commit_recovery(&state).is_err()); + drop(state); + assert_eq!(count.load(Ordering::SeqCst), 0); + assert_eq!( + runtime + .inner + .store + .aggregate_seq(&original.activity_id) + .unwrap(), + Some(2) + ); + let lookup = runtime + .snapshot_recovery_confirm(&session, &request.confirmation) + .await + .unwrap(); + assert_eq!(lookup.state, super::super::SnapshotRecoveryState::Unknown); + assert!(lookup.receipt.is_none()); + runtime.shutdown().await; +} + #[tokio::test] async fn late_snapshot_disposal_preserves_durable_acknowledgement_and_unrelated_uncertainty() { for (unrelated_uncertainty, other_snapshot) in [(false, false), (true, false), (false, true)] { @@ -194,11 +602,13 @@ async fn reload_preserves_generic_uncertainty_separately_from_snapshot_owners() .unwrap() .contains(&activity_id) ); - let mut generic = activity::Scope::reserve(&runtime.inner, &handle) - .await - .unwrap(); - generic.launch().unwrap(); - drop(generic); + assert!( + activity::Scope::reserve(&runtime.inner, &handle) + .await + .is_err(), + "snapshot fence admitted a new generic native scope" + ); + legacy_generic_unknown(&runtime, &handle, &session); assert!(handle.location_uncertain.load(Ordering::SeqCst)); let reloaded = restart(&runtime, temp.path()); let restored = reloaded.inner.handle(&session).await.unwrap(); @@ -344,6 +754,24 @@ async fn rejected_origin_append_cannot_authorize_source_effects_even_if_the_adap #[tokio::test] async fn snapshot_completion_commits_session_and_exact_origin_before_native_ack() { let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut unrelated = activity::Scope::reserve(&runtime.inner, &handle) + .await + .unwrap(); + let fake = SnapshotOrigin { + version: 1, + session_id: session.clone(), + session_created_id: "untrusted_attempt".into(), + activity_id: unrelated.id().to_owned(), + worktree_id: None, + binding: binding(temp.path().canonicalize().unwrap()), + }; + assert!( + unrelated.launch_snapshot(fake).is_err(), + "generic activity adopted a snapshot origin" + ); + unrelated.settle().unwrap(); + drop(unrelated); let owner = worker(&runtime, &session).await; let native_binding = binding(temp.path().canonicalize().unwrap()); let acknowledgements = Arc::new(AtomicUsize::new(0)); @@ -368,17 +796,12 @@ async fn snapshot_completion_commits_session_and_exact_origin_before_native_ack( .unwrap(); assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); let handle = runtime.inner.handle(&session).await.unwrap(); - let mut unrelated = activity::Scope::reserve(&runtime.inner, &handle) - .await - .unwrap(); - let unrelated_id = unrelated.id().to_owned(); - let origin = completed.worker.scope.snapshot_origin().unwrap().clone(); assert!( - unrelated.launch_snapshot(origin).is_err(), - "generic activity adopted a snapshot origin" + activity::Scope::reserve(&runtime.inner, &handle) + .await + .is_err() ); - unrelated.launch().unwrap(); - drop(unrelated); + let unrelated_id = legacy_generic_unknown(&runtime, &handle, &session); let mut state = handle.state.lock().await; completed .commit( diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 28340dfcf..c0d4cf245 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -72,6 +72,8 @@ pub(crate) struct WorkerGuard { pub store: Store, pub workspace: tokio::sync::OwnedMutexGuard<()>, pub origin: Option, + pub digest: Option, + pub receipt: Option, } impl cyber_server::runtime::SnapshotNativeGuard for WorkerGuard { @@ -80,6 +82,12 @@ impl cyber_server::runtime::SnapshotNativeGuard for WorkerGuard { if origin != self.origin.as_ref() { return Err("Snapshot receipt belongs to another worker".into()); } + if self.store.worker_digest(origin)? != self.digest { + return Err("Snapshot immutable intent changed before consumption".into()); + } + if let Some(receipt) = &self.receipt { + self.store.verify_receipt(origin, receipt)?; + } self.store.verify_worker_origin(origin) } fn acknowledge( diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs index 75c78865c..2865af446 100644 --- a/crates/cyber-snapshot/src/copy/resolution.rs +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -21,6 +21,15 @@ pub(super) struct Resolution { pub original_digest: String, pub review: Observations, pub outcome: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reviewed_state: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ReviewedState { + pub(super) completed: bool, + pub(super) acknowledged: bool, } impl Resolution { @@ -160,7 +169,125 @@ fn immutable_digest(record: &Journal) -> Result { )) } +fn fingerprint( + action: Action, + binding: &SnapshotBinding, + digest: &str, + observations: &Observations, + state: &ReviewedState, +) -> Result { + Ok(storage::digest( + &serde_json::to_vec(&( + 1u32, + action, + binding, + digest, + observations, + state.completed, + state.acknowledged, + )) + .map_err(fail)?, + )) +} + +fn verify_fingerprint(resolution: &Resolution) -> Result<()> { + let state = resolution + .reviewed_state + .as_ref() + .ok_or("Legacy resolution has no frozen review state")?; + if fingerprint( + resolution.request.action, + &resolution.original.binding, + &resolution.original_digest, + &resolution.review, + state, + )? != resolution.request.confirmation + { + return Err("Resolution differs from its confirmed immutable review".into()); + } + Ok(()) +} + impl Store { + pub(crate) fn worker_digest(&self, origin: Option<&SnapshotOrigin>) -> Result> { + origin + .map(|origin| { + let (_, journal) = self.journal(&origin.binding.journal_id)?; + if journal.origin.as_ref() != Some(origin) { + return Err("Snapshot native origin changed".into()); + } + immutable_digest(&journal) + }) + .transpose() + } + pub(super) fn verify_receipt( + &self, + origin: Option<&SnapshotOrigin>, + expected: &Receipt, + ) -> Result<()> { + let origin = origin.ok_or("Recovery completion has no reserved native worker origin")?; + let (_, journal) = self.journal(&origin.binding.journal_id)?; + if receipt(&origin.binding.journal_id, &journal)? != *expected { + return Err("Native resolution receipt changed".into()); + } + Ok(()) + } + pub(super) fn verify_settlement(&self, parent: &Journal) -> Result<()> { + let id = parent + .settled_by + .as_deref() + .ok_or("Missing native settlement reference")?; + let (_, journal) = self.journal(id)?; + let resolution = journal + .resolution + .as_ref() + .ok_or("Native settlement reference is not a resolution")?; + verify_fingerprint(resolution)?; + if !parent.acknowledged + || !journal.acknowledged + || !journal.completed + || resolution.outcome.is_none() + || parent.origin.as_ref() != Some(&resolution.original) + || immutable_digest(parent)? != resolution.original_digest + { + return Err("Native settlement reference differs from acknowledged resolution".into()); + } + Ok(()) + } + + /// Called only after the runtime consumed the exact two-owner receipt. This + /// deliberately checks immutable native proof, not later owner file edits. + pub(super) fn acknowledge_resolution( + &self, + origin: &SnapshotOrigin, + dir: Dir, + mut journal: Journal, + ) -> Result<()> { + let resolution = journal + .resolution + .as_ref() + .ok_or("Missing resolution intent")?; + verify_fingerprint(resolution)?; + let _receipt = receipt(&origin.binding.journal_id, &journal)?; + self.verify_worker_namespace(&resolution.original)?; + let (parent_dir, mut parent) = self.journal(&resolution.original.binding.journal_id)?; + if parent.origin.as_ref() != Some(&resolution.original) + || immutable_digest(&parent)? != resolution.original_digest + || parent + .settled_by + .as_ref() + .is_some_and(|id| id != &origin.binding.journal_id) + { + return Err("Original intent differs from consuming resolution acknowledgement".into()); + } + // The child acknowledgement is durable before publishing the parent's + // settlement link. A cut between writes stays fenced and is idempotent. + journal.acknowledged = true; + Self::record(&dir, &journal)?; + parent.acknowledged = true; + parent.settled_by = Some(origin.binding.journal_id.clone()); + Self::record(&parent_dir, &parent) + } pub(super) fn resolution_preimages<'a>( &self, resolution: Option<&'a Resolution>, @@ -296,17 +423,16 @@ impl Store { .and_then(|origin| origin.binding.lock_identity.clone()), }; if review.conflicts.is_empty() { - let raw = serde_json::to_vec(&( - 1u32, + review.confirmation = Some(fingerprint( action, &binding, &digest, &observations, - record.completed, - record.acknowledged, - )) - .map_err(fail)?; - review.confirmation = Some(storage::digest(&raw)); + &ReviewedState { + completed: record.completed, + acknowledged: record.acknowledged, + }, + )?); } if serde_json::to_vec(&review).map_err(fail)?.len() > 16 * 1024 * 1024 { return Err("Recovery review exceeds response limit".into()); @@ -365,6 +491,10 @@ impl Store { original_digest: plan.digest, review: plan.observations.clone(), outcome: None, + reviewed_state: Some(ReviewedState { + completed: parent.completed, + acknowledged: parent.acknowledged, + }), }; let mut record = self .prepare_journal( @@ -440,6 +570,7 @@ impl Store { let Some(resolution) = &record.resolution else { return Ok(()); }; + verify_fingerprint(resolution)?; self.verify_worker_namespace(&resolution.original)?; let (parent_dir, parent) = self.journal(&resolution.original.binding.journal_id)?; if immutable_digest(&parent)? != resolution.original_digest { diff --git a/crates/cyber-snapshot/src/copy/resolution_tests.rs b/crates/cyber-snapshot/src/copy/resolution_tests.rs index 7ff65a377..a477416db 100644 --- a/crates/cyber-snapshot/src/copy/resolution_tests.rs +++ b/crates/cyber-snapshot/src/copy/resolution_tests.rs @@ -45,6 +45,137 @@ impl Prepared { } } +#[tokio::test] +async fn immutable_intent_handoff_rejects_changed_context_with_unchanged_files_and_receipt() { + use cyber_server::runtime::SnapshotNativeGuard; + let p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "immutable-handoff"); + let receipt = p.resolve(&request); + let (dir, mut journal) = p.store.journal(&receipt.resolution_id).unwrap(); + let mut origin = p.record.origin.as_ref().unwrap().clone(); + origin.activity_id = "op_resolution_fixture".into(); + origin.binding.journal_id = receipt.resolution_id.clone(); + origin.binding.journal_identity = storage::dir_identity(&dir).unwrap(); + journal.origin = Some(origin.clone()); + Store::record(&dir, &journal).unwrap(); + let digest = p.store.worker_digest(Some(&origin)).unwrap(); + let workspace = std::sync::Arc::new(tokio::sync::Mutex::new(())) + .lock_owned() + .await; + let guard = crate::copy::WorkerGuard { + store: p.store, + workspace, + origin: Some(origin.clone()), + digest, + receipt: Some(receipt), + }; + guard.verify(Some(&origin)).unwrap(); + let snapshot = format!("copy:{}", "0".repeat(64)); + journal.context = Some(super::super::RestoreContext { + target: snapshot.clone(), + recorded: snapshot.clone(), + expected: snapshot, + exclusions: vec!["changed after handoff".into()], + }); + Store::record(&dir, &journal).unwrap(); + guard.store.verify_worker_origin(Some(&origin)).unwrap(); + assert!( + guard.verify(Some(&origin)).is_err(), + "native intent changed after its completion handoff" + ); + assert!( + !guard + .store + .journal("restore_fixture") + .unwrap() + .1 + .acknowledged + ); +} + +#[test] +fn changed_frozen_resolution_review_cannot_be_consumed_or_acknowledged() { + let p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "immutable-review"); + let receipt = p.resolve(&request); + let (dir, mut journal) = p.store.journal(&receipt.resolution_id).unwrap(); + let mut origin = p.record.origin.as_ref().unwrap().clone(); + origin.activity_id = "op_resolution_fixture".into(); + origin.binding.journal_id = receipt.resolution_id.clone(); + origin.binding.journal_identity = storage::dir_identity(&dir).unwrap(); + journal.origin = Some(origin.clone()); + Store::record(&dir, &journal).unwrap(); + let before = p.store.worker_digest(Some(&origin)).unwrap(); + journal + .resolution + .as_mut() + .unwrap() + .reviewed_state + .as_mut() + .unwrap() + .completed = true; + Store::record(&dir, &journal).unwrap(); + assert_ne!(p.store.worker_digest(Some(&origin)).unwrap(), before); + assert!(p.store.verify_worker_origin(Some(&origin)).is_err()); + assert!(p.store.acknowledge_worker(Some(&origin)).is_err()); + assert!(!p.store.journal("restore_fixture").unwrap().1.acknowledged); + assert!( + !p.store + .journal(&receipt.resolution_id) + .unwrap() + .1 + .acknowledged + ); +} + +#[test] +fn acknowledged_resolution_settles_incomplete_parent_without_replaying_later_owner_edits() { + let p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "settlement-link"); + let receipt = p.resolve(&request); + let (dir, mut resolution) = p.store.journal(&receipt.resolution_id).unwrap(); + let mut origin = p.record.origin.as_ref().unwrap().clone(); + origin.activity_id = "op_resolution_fixture".into(); + origin.binding.journal_id = receipt.resolution_id.clone(); + origin.binding.journal_identity = storage::dir_identity(&dir).unwrap(); + resolution.origin = Some(origin.clone()); + Store::record(&dir, &resolution).unwrap(); + // Model a cut after child acknowledgement but before the parent link. These + // synthetic origins do not confer runtime acknowledgement authority. + resolution.acknowledged = true; + Store::record(&dir, &resolution).unwrap(); + assert!(p.store.ensure_recovered().is_err()); + std::fs::write(p.root.join("existing/replace"), b"later owner edit").unwrap(); + assert!(p.store.verify_worker_origin(Some(&origin)).is_err()); + // After a durable consuming receipt, acknowledgement reconciliation checks + // immutable native evidence and must not revalidate/replay owner file edits. + p.store.acknowledge_worker(Some(&origin)).unwrap(); + let (_, parent) = p.store.journal("restore_fixture").unwrap(); + assert!(!parent.completed); + assert!(parent.acknowledged); + assert_eq!( + parent.settled_by.as_deref(), + Some(receipt.resolution_id.as_str()) + ); + p.store.ensure_recovered().unwrap(); + p.store.acknowledge_worker(Some(&origin)).unwrap(); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"later owner edit" + ); + assert_eq!(storage::read(&p.dir, "stage_0", 65536).unwrap(), b"before"); + let mut missing = parent; + missing.settled_by = Some("restore_missing_resolution".into()); + Store::record(&p.dir, &missing).unwrap(); + assert!( + p.store.ensure_recovered().is_err(), + "missing reference licensed native settlement" + ); +} + #[test] fn rollback_reverses_proven_files_and_retains_actual_displaced_live_inodes() { let p = Prepared::managed(); diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 7b70140d6..5bc95f19c 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -34,6 +34,10 @@ pub(super) struct Journal { pub origin: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub acknowledged: bool, + /// A failed restore remains incomplete. Only an acknowledged, independently + /// retained resolution can settle its native fence. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub settled_by: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub ownership: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -55,6 +59,10 @@ impl Store { pub(super) fn ensure_recovered(&self) -> Result<()> { for id in storage::names(&self.journals, ENTRY_LIMIT)? { let (_, record) = self.journal(&id)?; + if record.settled_by.is_some() { + self.verify_settlement(&record)?; + continue; + } if !record.completed || ((record.origin.is_some() || record.resolution.is_some()) && !record.acknowledged) @@ -87,6 +95,16 @@ impl Store { .all(|(path, change)| path == &change.path) || (record.completed && record.applied.len() != record.changes.len()) || (record.origin.is_none() && record.acknowledged) + || record.settled_by.as_ref().is_some_and(|resolution| { + !record.acknowledged + || record.origin.is_none() + || resolution == id + || resolution.len() > 256 + || !resolution.starts_with("restore_") + || !resolution + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + }) || (record.version < 3 && record.ownership.is_some()) || (record.version == 3 && record.ownership.is_none() @@ -464,6 +482,7 @@ impl Store { changes, origin, acknowledged: worker.is_some(), + settled_by: Some("0".repeat(256)), ownership, context, resolution, @@ -472,6 +491,18 @@ impl Store { if bytes > RECORD_LIMIT { return Err("Restore journal exceeds limit".into()); } + let parent_reserve = if let Some(resolution) = &journal.resolution { + let (_, mut parent) = self.journal(&resolution.original.binding.journal_id)?; + parent.acknowledged = true; + parent.settled_by = Some(id.to_owned()); + let bytes = serde_json::to_vec(&parent).map_err(fail)?.len() as u64; + if bytes > RECORD_LIMIT { + return Err("Original settlement journal exceeds limit".into()); + } + bytes.checked_mul(2).ok_or("Restore size overflow")? + } else { + 0 + }; let reserve = journal .changes .iter() @@ -483,13 +514,17 @@ impl Store { .max(change.after.as_ref().map_or(0, |entry| entry.length)) }) .try_fold( - bytes.checked_mul(2).ok_or("Restore size overflow")?, + bytes + .checked_mul(2) + .and_then(|bytes| bytes.checked_add(parent_reserve)) + .ok_or("Restore size overflow")?, |sum, length| sum.checked_add(length).ok_or("Restore size overflow"), )?; self.reserve(reserve)?; journal.applied.clear(); journal.origin = None; journal.acknowledged = false; + journal.settled_by = None; journal.ownership = None; if let Some(resolution) = &mut journal.resolution { resolution.outcome = None; @@ -564,6 +599,10 @@ impl Store { if journal.origin.as_ref() != Some(origin) || !journal.completed { return Err("Snapshot acknowledgement differs from its completed worker".into()); } + self.verify_worker_namespace(origin)?; + if journal.resolution.is_some() { + return self.acknowledge_resolution(origin, dir, journal); + } journal.acknowledged = true; Self::record(&dir, &journal) } @@ -801,6 +840,7 @@ mod tests { applied: vec![], origin: Some(origin.clone()), acknowledged: false, + settled_by: None, ownership: None, context: None, resolution: None, diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index dd40af192..97cec517f 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -295,12 +295,17 @@ impl Snapshots for GitSnapshots { .map_err(RestoreError::Failed)?; let receipt = store.resolve_recovery(&request, &|| Ok(()), Some(work))?; let origin = work.origin().cloned(); + let digest = store + .worker_digest(origin.as_ref()) + .map_err(RestoreError::Failed)?; let mut native = cyber_server::runtime::SnapshotNativeResult::new( vec![], Box::new(copy::WorkerGuard { store, workspace, origin, + digest, + receipt: Some(receipt.clone()), }), ); native.recovery = Some(receipt); @@ -363,12 +368,17 @@ impl Snapshots for GitSnapshots { Some(work), )?; let origin = work.origin().cloned(); + let digest = store + .worker_digest(origin.as_ref()) + .map_err(RestoreError::Failed)?; Ok(cyber_server::runtime::SnapshotNativeResult::new( paths, Box::new(copy::WorkerGuard { store, workspace, origin, + digest, + receipt: None, }), )) }) diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 76a0d9a33..4ac1d5ab5 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -699,3 +699,98 @@ the owned versus legacy commit paths and shared binary worker behavior. It corrected the potential post-commit flag race and combined-read ordering. The broader recovery authority, native consuming transaction and reconciliation gates remain required before operator apply or cleanup is admitted. + +### Non-Git recovery admission and consuming owner pair + +The file-copy recovery path now connects review to a durable request reservation, +the actual owned filesystem worker and the exact consuming native transaction. +`POST /sessions/{sessionID}/rewind/recovery/confirm` accepts the journal, action, +fingerprint and stable request ID. `GET .../recovery/requests/{requestID}` reads only +the durable request (404 when absent). OpenAPI and typed SDK methods are generated. +Existing request IDs return lookup before native access, including unknown outcomes +and later owner edits; different content cannot reuse the request. Lookup reports +journal acknowledgement as unobserved rather than inferring it from runtime +settlement. A request resolved by another request has a distinct result state. + +The reservation aggregate derives from Session creation and a length-framed request +ID hash, before checkout/native preparation. Snapshot-only recovery authority +captures ancestry, local/subtree revisions, active closed scopes and solve fence +revisions. It preserves those boundaries and cannot lend ordinary launch authority. +The Store projector validates the exact reservation and immutable launch metadata; +it does not grant generic hook/process admission through a snapshot fence. Ordinary +native admission and new ordinary snapshot admission are mutually fenced at the +writer. Prompt, child and tool admission targets also check the durable snapshot +fence there, so a stale caller precheck cannot race a newly committed reservation. +Regressions verify rejected native reservation and input append leave no published +ownership or inbox row. Recovery refuses live local native controls/background work. The +actual callback retains checkout/lifecycle/workspace/private-store ownership. + +After native guard revalidation, the writer pair checks both original/resolution +origins, the captured cancellation boundary, current Session revision and exact +request/receipt. Only an observed durable pair settles the original native owner +and records the resolution receipt, removing only the original snapshot reason. +It emits no old rewind or transcript events and preserves generic unknown outcomes. +Normal rewind commit cannot consume a recovery result or lend its authority. + +Native journal acknowledgement follows that observed writer commit. It publishes +the resolution acknowledgement first, then a validated settlement link in the +original journal. The original may remain `completed: false`; its history is never +rewritten as a completed restore. Missing/mismatched acknowledgement references +retain the fence. Metadata budgeting includes atomic replacement of the parent +settlement record, and no source directory or retained payload is reclaimed. + +Substantive code/security review corrected ordinary hook admission through the +snapshot fence and private intent changes during the consuming handoff. The native +worker now pins the immutable intent digest and exact filesystem receipt. The +resolution retains the reviewed completion/acknowledgement state, allowing its +fingerprint to be verified independently of later mutable acknowledgement flags. +Tests reject a changed frozen review and changed intent context even when live +files and the returned receipt are unchanged. Old resolutions without this frozen +review state remain readable but cannot acquire that missing consuming proof. +Origins without the exact native lock proof remain inspectable and fenced. + +Real CLI/API acceptance creates 1280 non-Git files, records two snapshots, observes +an actual source exchange during restore and kills the real service while the +namespace is partial. A fresh service observes both applied and unapplied source +files. Separate keep and rollback scenarios then confirm recovery, inspect the +durable settlement link, preserve unrelated owner data and the unchanged +conversation, and verify no additional scripted-provider inference. Keep also +preserves a later ordinary owner image. Retries after another owner edit and after +another service restart are lookup-only; changed request content is refused. This +uses actual file-copy operations, server, CLI and process kills, without a restore +fault hook. It covers one observed partial-exchange cut per action, not every cut. + +Runtime regressions preserve a captured closed boundary and unrelated generic +unknown evidence, reject a new stop boundary before effects, and reject a changed +Session revision before acknowledgement. The WAL-hook commit-reply-loss regression +uses a real file-backed Store: both owner settlements persist, no commit +notification or file acknowledgement is emitted, and lookup finds the same receipt +after the Store is closed and reopened. Its filesystem callback is instrumented; +it is not a service kill at the writer boundary or journal reconciliation proof. + +Final local macOS evidence: 404 server unit tests passed (one ignored subprocess +fixture invoked by its parent Location identity tests), four idle-cancellation, +thirteen Location and five rewind-preview integration cases passed. Snapshot +evidence includes 35 final unit cases, eighteen file-copy and seventeen Git cases +(70 unique tests). All fourteen CLI branch/recovery cases passed; the new real-kill +case was rerun after the helper refactor and added request-content refusal and +passed for both actions. SDK typecheck and generation checks passed; SDK tests +passed 130 cases with ten configured process end-to-end cases skipped because +`CYBER_BIN` was unset. OpenAPI consistency passed. Server/snapshot/CLI all-target +Clippy with warnings denied passed. The complexity skill has no Rust analyzer; +Clippy found the test helper at 32/15, and extracting its kill, settlement-link and +retry phases passed the repository threshold without allowances. Formatting and +diff checks passed. macOS sandbox deprecation and Darwin linker warnings were +observed without assigning their origin to main. + +The complete feature is not merge-ready. Explicit journal acknowledgement +reconciliation after lost replies or failed flushes is still absent. Reservations +that never publish a native origin remain unknown and lookup-only; missing records +do not prove no effects or authorize clearing checkout/process uncertainty. Those +resource/intent cuts require their own exact proof and operator resolution. Git +journaling, interrupted-resolution/pre-origin acceptance, reference/reader/lease +protection and two-phase reclamation, typed CLI/TUI/App recovery, full native cuts, +cross-platform results and complete delivery review remain open. Linux/Windows, +full-workspace checks and real SDK process acceptance were not run here. M2–M8 and +both issues remain open. A remote refresh still reports main `db9e171e` and no open +PRs; this increment has not been pushed or merged. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 6e553d462..9f7567aba 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -8,6 +8,13 @@ acknowledgement/reference checks. The earlier 2026-11-05 feature request alone did not select this protocol. Approval does not establish implementation or acceptance; the native/platform and complete recovery tests remain required. +Implementation checkpoint: the non-Git review/confirm/lookup API and SDK now use +durable request reservation and paired native settlement. Local real-service +partial-restore kills exercise keep/rollback and lookup without replay. Explicit +journal acknowledgement reconciliation, pre-origin resource cuts, Git journaling, +reclamation, complete clients and the platform matrix remain unfinished. Parent +tasks are open; the complete recovery delivery is not merge-ready. + ## Owner-visible contract The owner selects a retained restore by Session/Location and journal ID, requests a diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index cb1e4603d..f36ae2b6a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -36,31 +36,37 @@ recovery implementation. - [ ] M4 Implement bounded keep/rollback planning and fingerprint revalidation under exact native locks. Preserve unrelated edits, refuse the complete plan on conflicts, and retain any interrupted resolution as independent evidence. Test keep and rollback after actual partial namespace replacement, including staged undo and inactive branches. - [ ] M5 Atomically validate and commit the exact origin acknowledgement and resolution receipt at the Store writer. Determine the bounded owner aggregates before extending the single-aggregate append API. Test late CAS/projector/validation failure rollback, absence of uncommitted notifications and committed-but-lost writer replies; repeat request lookup must never mutate files again. -M5 prerequisite: `Store::append_pair_checked` implements the bounded two-owner -transaction with exact sequences, post-commit notifications and rollback/lost-ack -regressions. Native origin validation and resolution request ownership are not -implemented by that primitive; M5 remains open until the runtime uses it with -exact native proof and actual restart/retry acceptance. - -M4/M7 continuation: the read-only recovery review API and native file-copy -resolution backend now exist. The backend writes a separate intent and retains -actual displaced live inodes; existing request IDs are lookup-only. This is not -operator apply acceptance. Dedicated recovery admission, global durable request -ownership and the exact original/resolution writer pair remain required. Ordinary -rewind commit explicitly refuses a recovery receipt. No native fence is cleared by -backend filesystem completion. Created source directories remain protected for -acknowledged, ownership-checked cleanup. +M5 implementation now uses `Store::append_pair_checked` for the exact original +native owner and the resolution request owner. A Session-creation/request-derived +reservation precedes native work, and the consuming writer checks captured +recovery authority, current Session revision, both immutable origins and the exact +request/receipt. A WAL-hook reply-loss regression reopens the actual Store and +finds both committed owners without notifications or replay. M5 remains open for +complete failure/reconciliation and native acceptance; the primitive alone did not +satisfy these obligations. + +M4/M7 continuation: review, confirm and request lookup HTTP/SDK operations now +exist for the file-copy backend. Dedicated snapshot-only admission preserves +captured closed boundaries; ordinary native/hook admission is fenced at the writer. +Actual CLI/API service-kill cases resolve a partial namespace with keep and +rollback, preserve unrelated/later owner files and the transcript, and repeat only +lookup after restart. This is local non-Git evidence, not complete operator/client +or platform acceptance. Typed CLI/TUI recovery, explicit acknowledgement +reconciliation, pre-origin cuts and Git-native equivalents remain required. +Ordinary rewind commit still refuses recovery receipts. Filesystem completion alone +never clears a native fence. Created source directories remain protected for safe +acknowledged cleanup. Late original-scope disposal now checks durable identity and status within the writer transaction after native store-lock release. It preserves an already acknowledged activity and its complete receipt; changed ownership/sequence refuses -the append. This does not implement recovery acknowledgement. Before enabling M5, -preserve opaque/generic Location uncertainty. Snapshot uncertainty is now tracked +the append. The consuming recovery pair now removes only its exact snapshot +reason after observed durable commit, preserving opaque/generic Location +uncertainty. Snapshot uncertainty is tracked by exact activity ID separately from the opaque flag; reload reconstructs generic fences and snapshot owners independently. A late-scope no-op clears only its exact snapshot reason after observing the matching durable acknowledgement at the writer. -The consuming recovery commit still needs to apply this distinction. Reconciliation -must distinguish a durable writer +Explicit reconciliation still needs to distinguish a durable writer acknowledgement with missing journal acknowledgement from an uncommitted filesystem outcome, and must not revalidate or replay later owner edits as the old request. Unprepared reservation/origin/intent cuts still require exact resource ownership @@ -74,9 +80,10 @@ Implementation source map: The non-Git reviewed-confirm path now uses a runtime-issued synchronous worker, origin-bound intent and paired Session/native commit followed by journal -acknowledgement. M2–M8 remain open: Git and legacy entry-point integration, -managed keep/rollback and request lookup, resolution receipts, lost-reply journal -reconciliation, cleanup and complete native/platform acceptance are still required. +acknowledgement. The recovery path adds globally reserved request ownership and +paired original/resolution settlement. M2–M8 remain open: Git and legacy entry-point +integration, pre-origin resource proof, lost-reply journal reconciliation, cleanup, +complete clients and native/platform acceptance are still required. New version-3 file-copy intents retain native per-change source/staged file and parent identities, including privately staged identities for new parents before @@ -84,18 +91,20 @@ publication. Read-only inspection classifies actual source/private slots without using progress as proof; five storage-level process-kill cuts verify this after fresh ownership. Legacy version-1/2 intents have no historical per-file proof. Do not infer rollback ownership from matching bytes or missing progress. Git -equivalence, managed resolution intents/operations, runtime crash cuts and the -complete native platform matrix remain open with M3/M8. +equivalence, the remaining runtime/crash boundaries and the complete native platform +matrix remain open with M3/M8. Two real non-Git CLI/API scenarios now kill the +service after a partial exchange, restart, keep/rollback and verify lookup without +replay; these do not cover every cut or reclamation. Crash acceptance must also cover reservation before origin publication and origin publication before intent publication. Those durable fences cannot be force-cleared or resolved by inventing a missing journal; implement exact pre-effect ownership proof/reconciliation for these cuts before claiming complete managed recovery. - `cyber-server/src/runtime/activity.rs` owns native activity records; `binary_documents.rs` retains a real synchronous worker's resources and `binary_recovery.rs` demonstrates document-specific recovery proof. Snapshot workers require their own origin/proof rather than calling binary recovery. -- `cyber-server/src/runtime/host.rs::Snapshots` currently exposes restore and read-only recovery inspection; `rewind_preview.rs` owns reviewed confirmation. A dedicated owner recovery admission must work while ordinary admission remains fenced. +- `cyber-server/src/runtime/host.rs::Snapshots` exposes restore, recovery inspection and owned resolution; `rewind_preview.rs` owns reviewed rewind confirmation. `snapshot_recovery.rs` owns durable recovery admission/lookup, with snapshot-only captured authority and the paired native consuming commit in `activity.rs`. - `cyber-snapshot/src/copy/restore.rs` reads legacy version-1/2 intents and writes version-3 native ownership records. `copy/ownership.rs` prepares file/parent evidence and inspects actual slots; `copy/storage.rs` owns private descriptor-relative I/O. `restore.rs` and `git_history.rs` cover Git restore and historical exclusions. Keep historical evidence readable as the new protocol is added. -- `cyber-snapshot/src/copy/resolution.rs` freezes native observations, reads actual retained preimages, publishes independent keep/rollback intents and rejects request replay. Runtime `snapshot_recovery.rs` attests the selected Session/activity for read-only review; its consuming admission/commit is still required. Origins now bind the exact private lock inode; older origins without that proof remain inspectable and fenced. -- `cyber-store/src/store.rs::append_checked` currently appends one aggregate. Separate independent appends cannot establish atomic acknowledgement plus resolution ownership. +- `cyber-snapshot/src/copy/resolution.rs` freezes native observations, reads retained preimages, publishes independent intents and rejects request replay. Acknowledged child journals settle incomplete parents through validated links without inventing original completion. The retained worker guard pins immutable intent/receipt data. Origins bind the exact private lock inode; older origins without that proof remain inspectable and fenced. +- `cyber-store/src/store.rs::append_pair_checked` commits the bounded original/resolution owner pair. Independent single-owner appends cannot establish this atomic acknowledgement. ## 4. Rules and context inspection diff --git a/sdk/openapi.json b/sdk/openapi.json index c036106da..e13f3b1c5 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7353,6 +7353,131 @@ "gid" ] }, + "SnapshotRecoveryConfirmation": { + "type": "object", + "additionalProperties": false, + "properties": { + "journal_id": { + "type": "string" + }, + "action": { + "$ref": "#/components/schemas/SnapshotRecoveryAction" + }, + "confirmation": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "journal_id", + "action", + "confirmation", + "request_id" + ] + }, + "Data_SnapshotRecoveryResult": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/SnapshotRecoveryResult" + } + }, + "required": [ + "data" + ] + }, + "SnapshotRecoveryResult": { + "type": "object", + "properties": { + "request": { + "$ref": "#/components/schemas/SnapshotRecoveryConfirmation" + }, + "original": { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + "resolution": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + { + "type": "null" + } + ] + }, + "state": { + "$ref": "#/components/schemas/SnapshotRecoveryState" + }, + "receipt": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRecoveryReceipt" + }, + { + "type": "null" + } + ] + }, + "journal_acknowledged": { + "type": [ + "boolean", + "null" + ], + "description": "Lookup does not inspect native files. None means journal acknowledgement\nhas not been observed by this response, even if runtime settlement is durable." + } + }, + "required": [ + "request", + "original", + "state" + ] + }, + "SnapshotRecoveryState": { + "type": "string", + "enum": [ + "pending", + "unknown", + "acknowledged", + "resolved_by_another_request" + ] + }, + "SnapshotRecoveryReceipt": { + "type": "object", + "properties": { + "journal_id": { + "type": "string" + }, + "resolution_id": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "action": { + "$ref": "#/components/schemas/SnapshotRecoveryAction" + }, + "confirmation": { + "type": "string" + }, + "original": { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + "filesystem_complete": { + "type": "boolean" + } + }, + "required": [ + "journal_id", + "resolution_id", + "request_id", + "action", + "confirmation", + "original", + "filesystem_complete" + ] + }, "RewindSelection": { "type": "object", "additionalProperties": false, @@ -24272,6 +24397,114 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/recovery/confirm": { + "post": { + "operationId": "v1.session.snapshotRecoveryConfirm", + "summary": "Resolve fingerprint-confirmed native snapshot effects with an exact durable request; repeated IDs are lookup-only", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotRecoveryResult" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SnapshotRecoveryConfirmation" + } + } + } + } + } + }, + "/api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}": { + "get": { + "operationId": "v1.session.snapshotRecoveryLookup", + "summary": "Look up a durable recovery request without native access, acknowledgement reconciliation, replay or inference", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "requestID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotRecoveryResult" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/preview": { "post": { "operationId": "v1.session.rewindPreview", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index 86967e94f..b342b1d76 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1559,6 +1559,18 @@ export interface Operations { body: undefined; response: S.SnapshotRecovery[]; }; + "v1.session.snapshotRecoveryConfirm": { + path: { sessionID: string }; + query: Record; + body: S.SnapshotRecoveryConfirmation; + response: S.SnapshotRecoveryResult; + }; + "v1.session.snapshotRecoveryLookup": { + path: { sessionID: string; requestID: string }; + query: Record; + body: undefined; + response: S.SnapshotRecoveryResult; + }; "v1.session.snapshotRecoveryReview": { path: { sessionID: string }; query: Record; @@ -4650,6 +4662,30 @@ export const operations = { unwrap: true, stream: false, }, + "v1.session.snapshotRecoveryConfirm": { + tag: "session", + name: "snapshotRecoveryConfirm", + method: "POST", + path: "/api/v1/sessions/{sessionID}/rewind/recovery/confirm", + pathParams: ["sessionID"], + query: [], + body: "required", + located: false, + unwrap: true, + stream: false, + }, + "v1.session.snapshotRecoveryLookup": { + tag: "session", + name: "snapshotRecoveryLookup", + method: "GET", + path: "/api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}", + pathParams: ["sessionID", "requestID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, "v1.session.snapshotRecoveryReview": { tag: "session", name: "snapshotRecoveryReview", @@ -6055,6 +6091,12 @@ export function createGroups(caller: Caller) { /** Inspect retained filesystem restore journals without acknowledging or replaying native effects (`GET /api/v1/sessions/{sessionID}/rewind/recovery`) */ snapshotRecovery: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecovery", { path: { sessionID } }, options), + /** Resolve fingerprint-confirmed native snapshot effects with an exact durable request; repeated IDs are lookup-only (`POST /api/v1/sessions/{sessionID}/rewind/recovery/confirm`) */ + snapshotRecoveryConfirm: (sessionID: string, body: Operations["v1.session.snapshotRecoveryConfirm"]["body"], options?: RequestOptions): Promise => + caller.call("v1.session.snapshotRecoveryConfirm", { path: { sessionID }, body }, options), + /** Look up a durable recovery request without native access, acknowledgement reconciliation, replay or inference (`GET /api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}`) */ + snapshotRecoveryLookup: (sessionID: string, requestID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotRecoveryLookup", { path: { sessionID, requestID } }, options), /** Review retained snapshot effects without restoring, acknowledging native activity or starting inference (`POST /api/v1/sessions/{sessionID}/rewind/recovery/review`) */ snapshotRecoveryReview: (sessionID: string, body: Operations["v1.session.snapshotRecoveryReview"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecoveryReview", { path: { sessionID }, body }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 0507c1126..551f67bdb 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -1106,6 +1106,10 @@ export type Data_SideAnswer = { data: SideAnswer; }; +export type Data_SnapshotRecoveryResult = { + data: SnapshotRecoveryResult; +}; + export type Data_SnapshotRecoveryReview = { data: SnapshotRecoveryReview; }; @@ -4720,6 +4724,13 @@ export type SnapshotRecovery = { export type SnapshotRecoveryAction = "keep" | "rollback"; +export type SnapshotRecoveryConfirmation = { + journal_id: string; + action: SnapshotRecoveryAction; + confirmation: string; + request_id: string; +}; + export type SnapshotRecoveryImage = { hash: string; bytes: number; @@ -4743,6 +4754,29 @@ export type SnapshotRecoveryPath = { changes: boolean; }; +export type SnapshotRecoveryReceipt = { + journal_id: string; + resolution_id: string; + request_id: string; + action: SnapshotRecoveryAction; + confirmation: string; + original: SnapshotOrigin; + filesystem_complete: boolean; +}; + +export type SnapshotRecoveryResult = { + request: SnapshotRecoveryConfirmation; + original: SnapshotOrigin; + resolution?: SnapshotOrigin | null; + state: SnapshotRecoveryState; + receipt?: SnapshotRecoveryReceipt | null; + /** + * Lookup does not inspect native files. None means journal acknowledgement + * has not been observed by this response, even if runtime settlement is durable. + */ + journal_acknowledged?: boolean | null; +}; + export type SnapshotRecoveryReview = { journal_id: string; action: SnapshotRecoveryAction; @@ -4764,6 +4798,8 @@ export type SnapshotRecoverySelection = { action: SnapshotRecoveryAction; }; +export type SnapshotRecoveryState = "pending" | "unknown" | "acknowledged" | "resolved_by_another_request"; + export type SolveCandidateEvidence = { candidate_id: string; label?: string | null; diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index 9f4a618dd..86b4e8f40 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -31,3 +31,18 @@ test("snapshot recovery review preserves owner scope and honest acknowledgement assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/review"); assert.deepEqual(JSON.parse(calls[0]!.body!), { journal_id: "restore_one", action: "keep" }); }); + +test("snapshot recovery confirmation and lookup retain the exact request without implying journal acknowledgement", async () => { + const request = { journal_id: "restore_one", action: "keep" as const, confirmation: "a".repeat(64), request_id: "request/with space" }; + const result = { request, original: {}, resolution: null, state: "unknown", receipt: null, journal_acknowledged: null }; + const { client, calls } = mockClient(() => json(200, { data: result })); + assert.equal((await client.session.snapshotRecoveryConfirm("ses_owner", request)).state, "unknown"); + const lookup = await client.session.snapshotRecoveryLookup("ses_owner", request.request_id); + assert.equal(lookup?.journal_acknowledged, null); + assert.equal(calls.length, 2); + assert.equal(calls[0]!.method, "POST"); + assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/confirm"); + assert.deepEqual(JSON.parse(calls[0]!.body!), request); + assert.equal(calls[1]!.method, "GET"); + assert.equal(calls[1]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/requests/request%2Fwith%20space"); +}); From 354e5cac7be3f951a3f0a456b50aec39226b044a Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 20:27:55 -0400 Subject: [PATCH 08/87] Reconcile committed recovery journal acknowledgements --- crates/cyber-cli/tests/branches_cli.rs | 56 +++++ crates/cyber-server/src/http/openapi.rs | 2 + crates/cyber-server/src/http/sessions.rs | 15 ++ crates/cyber-server/src/runtime/host.rs | 12 + crates/cyber-server/src/runtime/mod.rs | 2 + .../src/runtime/snapshot_acknowledgement.rs | 236 ++++++++++++++++++ .../src/runtime/snapshot_recovery.rs | 42 ++++ .../src/runtime/snapshot_recovery_types.rs | 4 + .../src/runtime/snapshot_work_tests.rs | 183 +++++++++++++- crates/cyber-snapshot/src/copy/resolution.rs | 51 +++- .../src/copy/resolution_tests.rs | 106 ++++++++ crates/cyber-snapshot/src/lib.rs | 29 +++ .../implementation.md | 69 ++++- .../managed-recovery.md | 9 + .../complete-daily-coding-surfaces/tasks.md | 16 +- sdk/openapi.json | 64 +++++ sdk/typescript/src/generated/operations.ts | 21 ++ sdk/typescript/src/generated/types.ts | 5 + sdk/typescript/test/rewind.test.ts | 12 + 19 files changed, 922 insertions(+), 12 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_acknowledgement.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index e5a7594bc..8af2abae5 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -968,6 +968,62 @@ fn assert_recovery_lookup_does_not_replay( Some("new owner edit after recovery") ); assert_eq!(env.messages(id).as_slice(), messages); + reconcile_missing_journal_acknowledgement(env, id, request_id, result); + assert_eq!(env.messages(id).as_slice(), messages); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn reconcile_missing_journal_acknowledgement( + env: &Env, + id: &str, + request_id: &str, + result: &Value, +) { + let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + // Reproduce a cut after the durable owner pair but before either native ack. + // Only mutable acknowledgement metadata is reset; the persisted digest stays valid. + for journal in [ + &result["receipt"]["journal_id"], + &result["receipt"]["resolution_id"], + ] { + let path = namespace + .join("journals") + .join(journal.as_str().unwrap()) + .join("intent.json"); + let mut intent: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + intent["acknowledged"] = false.into(); + intent.as_object_mut().unwrap().remove("settled_by"); + std::fs::write(path, serde_json::to_vec(&intent).unwrap()).unwrap(); + } + let target = env.root.join("ack-owner-target"); + std::fs::write(&target, "new owner edit after recovery").unwrap(); + let source = env.root.join("repo/bulk_0000"); + std::fs::remove_file(&source).unwrap(); + std::os::unix::fs::symlink(&target, &source).unwrap(); + let path = format!("/sessions/{id}/rewind/recovery/requests/{request_id}/acknowledge"); + let reconciled = json(&env.cyber(&["api", "POST", &path, "-d", "{}"]))["data"].clone(); + assert_eq!(reconciled["receipt"], result["receipt"]); + assert_eq!(reconciled["journal_acknowledged"], true); + assert!( + std::fs::symlink_metadata(source) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_to_string(target).unwrap(), + "new owner edit after recovery" + ); + assert_native_settlement_link( + env, + result["receipt"]["journal_id"].as_str().unwrap(), + result, + ); } #[test] diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index 3082f725f..2d8593c92 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -768,6 +768,8 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { .ok::>(g), op("get", "/sessions/{sessionID}/rewind/recovery/requests/{requestID}", "v1.session.snapshotRecoveryLookup", "Look up a durable recovery request without native access, acknowledgement reconciliation, replay or inference") .ok::>(g), + op("post", "/sessions/{sessionID}/rewind/recovery/requests/{requestID}/acknowledge", "v1.session.snapshotRecoveryAcknowledge", "Reconcile journal acknowledgement for the exact committed native pair without replaying source changes or starting inference") + .ok::>(g), op("post", "/sessions/{sessionID}/rewind/preview", "v1.session.rewindPreview", "Review a rewind without changing files or the active conversation") .body::(g) .ok::>(g), diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index e4c10225d..19279e39d 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -358,6 +358,10 @@ pub fn routes() -> Router { &format!("{s}/rewind/recovery/requests/{{request_id}}"), get(snapshot_recovery_lookup), ) + .route( + &format!("{s}/rewind/recovery/requests/{{request_id}}/acknowledge"), + post(snapshot_recovery_acknowledge), + ) .route(&format!("{s}/rewind/confirm"), post(rewind_confirm)) .route(&format!("{s}/revert/stage"), post(stage)) .route(&format!("{s}/revert/clear"), post(clear)) @@ -1250,6 +1254,17 @@ async fn snapshot_recovery_lookup( })?, })) } +async fn snapshot_recovery_acknowledge( + State(state): State, + Path((id, request_id)): Path<(String, String)>, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_recovery_acknowledge(&id, &request_id) + .await?, + })) +} async fn rewind_preview( State(state): State, Path(id): Path, diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index b239a1eb6..3af01bb77 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -1445,6 +1445,18 @@ pub trait Snapshots: Send + Sync { ) -> BoxFuture<'_, Result> { Box::pin(async { Err(RestoreError::Failed("Owned recovery is unavailable".into())) }) } + /// Metadata-only reconciliation of an already committed native owner pair. + /// The actual synchronous callback must retain the supplied worker and may + /// not install source files, create a restore intent or launch subprocesses. + fn acknowledge_recovery( + &self, + _directory: String, + _worker: super::SnapshotAcknowledgementWorker, + ) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async { + Err("Native snapshot acknowledgement reconciliation is unavailable".into()) + }) + } /// Only synchronous filesystem hosts retaining runtime-issued worker ownership. fn owned_restore_available(&self, _directory: &str) -> bool { false diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index acf322071..b8b1c19d6 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -107,7 +107,9 @@ mod mcp_depth; pub mod messaging; pub mod peers; pub mod presence; +mod snapshot_acknowledgement; mod snapshot_recovery_authority; +pub use snapshot_acknowledgement::{SnapshotAcknowledgementWork, SnapshotAcknowledgementWorker}; mod snapshot_recovery_record; pub use mcp_depth::RunMarker; mod memory; diff --git a/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs new file mode 100644 index 000000000..257f56def --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs @@ -0,0 +1,236 @@ +//! Reconcile metadata for an already committed owner pair. No source-file capability. +use super::snapshot_recovery_authority::RecoveryAuthority; +use super::{ + Handle, Runtime, RuntimeError, SessionState, SnapshotOrigin, SnapshotRecoveryReceipt, activity, +}; +use std::sync::{Arc, PoisonError, atomic::Ordering}; +use tokio::sync::OwnedRwLockReadGuard; + +pub struct SnapshotAcknowledgementWorker { + runtime: Runtime, + authority: RecoveryAuthority, + sequence: i64, + receipt: SnapshotRecoveryReceipt, + origin: SnapshotOrigin, + control: Arc, + // Actual resources are released before ordinary admission or Stop waiters. + _location: Box, + _lifecycle: OwnedRwLockReadGuard<()>, + _admission: Admission, + _completion: Completion, +} +struct Admission(Arc); +impl Drop for Admission { + fn drop(&mut self) { + self.0.snapshot_active.store(false, Ordering::SeqCst); + } +} +struct Completion { + runtime: Runtime, + id: String, + control: Arc, +} +impl Drop for Completion { + fn drop(&mut self) { + self.runtime + .inner + .activities + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&self.id); + self.control.done.cancel(); + } +} + +/// Only acknowledgement metadata may be changed. This does not expose source +/// installation, intent creation, generic launch authority or worker-origin adoption. +pub struct SnapshotAcknowledgementWork<'a>(&'a SnapshotAcknowledgementWorker); +impl SnapshotAcknowledgementWork<'_> { + pub fn receipt(&self) -> &SnapshotRecoveryReceipt { + &self.0.receipt + } + pub fn origin(&self) -> &SnapshotOrigin { + &self.0.origin + } + pub fn authorize(&self) -> Result<(), String> { + if self.0.control.stop.is_cancelled() { + return Err("Snapshot acknowledgement was cancelled".into()); + } + let authority = self.0.authority.clone(); + let receipt = self.0.receipt.clone(); + let origin = self.0.origin.clone(); + let sequence = self.0.sequence; + self.0 + .runtime + .inner + .store + .read(move |db| { + let tx = db.unchecked_transaction()?; + verify_pair(&tx, &receipt, &origin)?; + let current: i64 = tx.query_row( + "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", + [&origin.session_id], + |row| row.get(0), + )?; + if current != sequence || !authority.current(&tx)? { + return Err(cyber_store::StoreError::InvalidAppend( + "Snapshot acknowledgement authority changed".into(), + )); + } + tx.commit()?; + Ok(()) + }) + .map_err(|error| error.to_string()) + } +} + +impl SnapshotAcknowledgementWorker { + pub async fn run( + self, + work: impl FnOnce(&SnapshotAcknowledgementWork<'_>) -> Result<(), String> + Send + 'static, + ) -> Result<(), String> { + let cancelled = self.control.stop.clone().drop_guard(); + let joined = tokio::task::spawn_blocking(move || work(&SnapshotAcknowledgementWork(&self))); + let result = joined.await.map_err(|_| { + "Snapshot acknowledgement worker failed; lookup and reconcile its committed receipt" + .to_owned() + })?; + cancelled.disarm(); + result + } +} + +impl Runtime { + pub(super) async fn snapshot_acknowledgement_worker( + &self, + handle: &Arc, + state: &SessionState, + receipt: SnapshotRecoveryReceipt, + origin: SnapshotOrigin, + ) -> Result { + let lifecycle = self.memory_mutation_lease().await?; + self.inner.ensure_idle(&state.info.id)?; + self.ensure_no_background(&state.info.id)?; + state.ensure_worktree_ready()?; + if self + .inner + .activities + .lock() + .unwrap_or_else(PoisonError::into_inner) + .values() + .any(|control| control.source == state.info.id) + { + return Err(RuntimeError::Busy(state.info.id.clone())); + } + let directory = std::fs::canonicalize(&state.info.directory) + .map_err(|error| RuntimeError::Invalid(error.to_string()))?; + if origin.session_id != state.info.id + || origin.worktree_id != state.info.worktree_id + || origin.binding.directory != directory + { + return Err(RuntimeError::Conflict( + "Committed acknowledgement belongs to another Location".into(), + )); + } + let authority = RecoveryAuthority::capture(self, &state.info.id)?; + let bound_receipt = receipt.clone(); + let bound_origin = origin.clone(); + self.inner.store.read(move |db| { + let tx = db.unchecked_transaction()?; + verify_pair(&tx, &bound_receipt, &bound_origin)?; + tx.commit()?; + Ok(()) + })?; + handle + .snapshot_active + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .map_err(|_| RuntimeError::Busy(state.info.id.clone()))?; + let admission = Admission(handle.clone()); + let control = Arc::new(activity::Control { + source: state.info.id.clone(), + binary_document: None, + snapshot_worker: true, + admission_sessions: super::admission_authority::binding_sessions(authority.bindings()), + stop: self.inner.closed.child_token(), + done: tokio_util::sync::CancellationToken::new(), + }); + let id = cyber_core::ids::new_id("op_ack"); + // This control is intentionally volatile: source effects already have + // their durable paired owners. A crash can leave only idempotent journal + // acknowledgement work, never a new source operation or orphan intent. + self.inner + .activities + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(id.clone(), control.clone()); + let completion = Completion { + runtime: self.clone(), + id, + control: control.clone(), + }; + let lease = self + .inner + .claim_location(&state.info, false, control.stop.clone()) + .await?; + if lease.worktree_id != state.info.worktree_id { + return Err(RuntimeError::Conflict( + "Acknowledgement checkout changed".into(), + )); + } + Ok(SnapshotAcknowledgementWorker { + runtime: self.clone(), + authority, + sequence: state.last_seq, + receipt, + origin, + control, + _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, + _lifecycle: lifecycle, + _admission: admission, + _completion: completion, + }) + } +} + +/// Observe both durable owner records in one read snapshot, including after a +/// committed-but-lost writer reply. Filesystem completion alone never passes. +fn verify_pair( + db: &rusqlite::Connection, + receipt: &SnapshotRecoveryReceipt, + origin: &SnapshotOrigin, +) -> Result<(), cyber_store::StoreError> { + let (_, original) = super::snapshot_recovery_record::attest(db, &receipt.original)?; + let (_, resolution) = super::snapshot_recovery_record::attest(db, origin)?; + let created: String = db.query_row( + "SELECT id FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq LIMIT 1", + [&origin.session_id, super::events::CREATED], + |row| row.get(0), + )?; + let request = resolution.snapshot_request.as_ref().ok_or_else(|| { + cyber_store::StoreError::InvalidAppend("Missing committed request".into()) + })?; + if original.status != "settled" + || resolution.status != "settled" + || original.phase != "snapshot_recovery_committed" + || resolution.phase != "snapshot_recovery_committed" + || original.snapshot_receipt.as_ref() != Some(receipt) + || resolution.snapshot_receipt.as_ref() != Some(receipt) + || created != origin.session_created_id + || created != receipt.original.session_created_id + || origin.activity_id + != super::snapshot_recovery::request_owner(&created, &receipt.request_id) + || origin.binding.journal_id != receipt.resolution_id + || receipt.journal_id != receipt.original.binding.journal_id + || request.original != receipt.original + || request.confirmation.request_id != receipt.request_id + || request.confirmation.journal_id != receipt.journal_id + || request.confirmation.action != receipt.action + || request.confirmation.confirmation != receipt.confirmation + || !receipt.filesystem_complete + { + return Err(cyber_store::StoreError::InvalidAppend( + "Journal acknowledgement requires the exact committed native pair".into(), + )); + } + Ok(()) +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index 023f8d221..31625a712 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -3,6 +3,48 @@ use super::{Runtime, RuntimeError, SnapshotRecoveryReview, SnapshotRecoverySelec use rusqlite::OptionalExtension; impl Runtime { + /// Reconcile only acknowledgement metadata for a durably committed request. + /// Unknown/incomplete requests cannot use this path to acquire an acknowledgement. + pub async fn snapshot_recovery_acknowledge( + &self, + session: &str, + request_id: &str, + ) -> Result { + let mut result = self + .snapshot_recovery_lookup(session, request_id) + .await? + .ok_or_else(|| RuntimeError::Invalid("No retained recovery request".into()))?; + if result.state != super::SnapshotRecoveryState::Acknowledged { + return Err(RuntimeError::Conflict( + "Only an exact committed recovery receipt can reconcile journal acknowledgement" + .into(), + )); + } + let receipt = result + .receipt + .clone() + .ok_or_else(|| RuntimeError::Corrupt("Committed recovery lacks its receipt".into()))?; + let origin = result.resolution.clone().ok_or_else(|| { + RuntimeError::Corrupt("Committed recovery lacks its native origin".into()) + })?; + let handle = self.inner.handle(session).await?; + let state = handle.state.lock().await; + let directory = state.info.directory.clone(); + let worker = self + .snapshot_acknowledgement_worker(&handle, &state, receipt.clone(), origin.clone()) + .await?; + drop(state); + self.inner + .options + .snapshots + .acknowledge_recovery(directory, worker) + .await + .map_err(RuntimeError::Conflict)?; + handle.acknowledge_snapshot_uncertainty(&receipt.original.activity_id); + handle.acknowledge_snapshot_uncertainty(&origin.activity_id); + result.journal_acknowledged = Some(true); + Ok(result) + } /// A repeated request is always lookup-only, even after a partial outcome or /// later owner edits. Reconciliation is a separate operation. pub async fn snapshot_recovery_confirm( diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs index 09f70f7cd..4ad5afddd 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs @@ -75,6 +75,10 @@ pub struct SnapshotRecoveryReceipt { pub confirmation: String, pub original: super::SnapshotOrigin, pub filesystem_complete: bool, + /// Immutable native resolution data, excluding acknowledgement/progress. + /// Missing on older receipts; those cannot authorize new journal writes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution_digest: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index e28537cec..3ca1c00c6 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -1,5 +1,5 @@ //! Instrumented worker ownership; native journal/namespace proof is tested separately. -use super::super::{Admission, Delivery, RuntimeOptions, events}; +use super::super::{Admission, Delivery, RuntimeOptions, Snapshots, events}; use super::*; use std::sync::atomic::AtomicUsize; use std::time::Duration; @@ -60,6 +60,14 @@ fn runtime_with_store( runtime: &Runtime, store: Arc, root: &std::path::Path, +) -> Runtime { + runtime_with_snapshots(runtime, store, root, Arc::new(super::super::NoSnapshots)) +} +fn runtime_with_snapshots( + runtime: &Runtime, + store: Arc, + root: &std::path::Path, + snapshots: Arc, ) -> Runtime { Runtime::new(RuntimeOptions { store, @@ -73,10 +81,60 @@ fn runtime_with_store( max_steps: None, today: None, interactive: false, - snapshots: Arc::new(super::super::NoSnapshots), + snapshots, }) } +struct Acknowledgements(Arc); +impl Snapshots for Acknowledgements { + fn track( + &self, + _: &str, + ) -> futures::future::BoxFuture<'_, Result, String>> { + Box::pin(async { Ok(None) }) + } + fn changed( + &self, + _: &str, + _: &str, + _: &str, + ) -> futures::future::BoxFuture<'_, Result, String>> { + Box::pin(async { Ok(vec![]) }) + } + fn diff( + &self, + _: &str, + _: &str, + _: &str, + ) -> futures::future::BoxFuture<'_, Result, String>> { + Box::pin(async { Ok(vec![]) }) + } + fn restore( + &self, + _: &str, + _: &str, + _: &str, + ) -> futures::future::BoxFuture<'_, Result, RestoreError>> { + Box::pin(async { panic!("acknowledgement replayed a source restore") }) + } + fn acknowledge_recovery( + &self, + _: String, + worker: super::super::SnapshotAcknowledgementWorker, + ) -> futures::future::BoxFuture<'_, Result<(), String>> { + let count = self.0.clone(); + Box::pin(async move { + worker + .run(move |work| { + work.authorize()?; + count.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .await + }) + } +} + #[tokio::test] async fn recovery_pair_lost_commit_reply_is_lookup_only_after_reopening_the_real_store() { fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { @@ -92,7 +150,9 @@ async fn recovery_pair_lost_commit_reply_is_lookup_only_after_reopening_the_real Runtime::registry(), )) .unwrap(); - let runtime = runtime_with_store(&base, Arc::new(store), temp.path()); + let acknowledged = Arc::new(AtomicUsize::new(0)); + let snapshots = Arc::new(Acknowledgements(acknowledged.clone())); + let runtime = runtime_with_snapshots(&base, Arc::new(store), temp.path(), snapshots.clone()); let session = runtime .create_session(super::super::CreateSession { directory: temp.path().to_str().unwrap().into(), @@ -113,6 +173,7 @@ async fn recovery_pair_lost_commit_reply_is_lookup_only_after_reopening_the_real drop(state); let count = Arc::new(AtomicUsize::new(0)); let completed = finish_recovery(owner, request.clone(), count.clone()).await; + legacy_generic_unknown(&runtime, &handle, &session); runtime .inner .store @@ -149,6 +210,29 @@ async fn recovery_pair_lost_commit_reply_is_lookup_only_after_reopening_the_real ); assert_eq!(lookup.journal_acknowledged, None); let receipt = lookup.receipt.unwrap(); + assert!( + handle + .snapshot_uncertain + .lock() + .unwrap() + .contains(&original.activity_id) + ); + let reconciled = runtime + .snapshot_recovery_acknowledge(&session, &request.confirmation.request_id) + .await + .unwrap(); + assert_eq!(reconciled.journal_acknowledged, Some(true)); + assert_eq!(acknowledged.load(Ordering::SeqCst), 1); + assert!(handle.snapshot_uncertain.lock().unwrap().is_empty()); + assert!( + handle.location_uncertain.load(Ordering::SeqCst), + "reconciliation cleared unrelated generic uncertainty" + ); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "reconciliation replayed the source worker" + ); runtime.shutdown().await; drop(handle); drop(runtime); @@ -157,7 +241,7 @@ async fn recovery_pair_lost_commit_reply_is_lookup_only_after_reopening_the_real Runtime::registry(), )) .unwrap(); - let reloaded = runtime_with_store(&base, Arc::new(reopened), temp.path()); + let reloaded = runtime_with_snapshots(&base, Arc::new(reopened), temp.path(), snapshots); let lookup = reloaded .snapshot_recovery_confirm(&session, &request.confirmation) .await @@ -232,6 +316,7 @@ async fn finish_recovery( confirmation: request.confirmation.confirmation, original: request.original, filesystem_complete: true, + resolution_digest: Some("d".repeat(64)), }); Ok(native) }) @@ -239,6 +324,95 @@ async fn finish_recovery( .unwrap() } +#[tokio::test] +async fn acknowledgement_cannot_adopt_an_uncommitted_filesystem_receipt() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let count = Arc::new(AtomicUsize::new(0)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Acknowledgements(count.clone())), + ); + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let request = recovery_request(original); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + drop(state); + let completed = finish_recovery(owner, request.clone(), Arc::new(AtomicUsize::new(0))).await; + drop(completed); + assert!( + runtime + .snapshot_recovery_acknowledge(&session, &request.confirmation.request_id) + .await + .is_err() + ); + assert_eq!(count.load(Ordering::SeqCst), 0); + assert!(handle.has_location_uncertainty()); + runtime.shutdown().await; + base.shutdown().await; +} + +#[tokio::test] +async fn disposed_acknowledgement_request_retains_actual_worker_resources_through_shutdown() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let request = recovery_request(original); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + drop(state); + let completed = finish_recovery(owner, request.clone(), Arc::new(AtomicUsize::new(0))).await; + let origin = completed.worker.scope.snapshot_origin().unwrap().clone(); + let state = handle.state.lock().await; + let receipt = completed.commit_recovery(&state).unwrap(); + let worker = runtime + .snapshot_acknowledgement_worker(&handle, &state, receipt, origin) + .await + .unwrap(); + drop(state); + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = permits.clone().acquire_owned().await.unwrap(); + let (started, running) = tokio::sync::oneshot::channel(); + let (finish, gate) = std::sync::mpsc::channel(); + let task = tokio::spawn(worker.run(move |work| { + let _permit = permit; + work.authorize()?; + started.send(()).unwrap(); + gate.recv().unwrap(); + assert!(work.authorize().is_err()); + Err("cancelled metadata worker".into()) + })); + running.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(permits.available_permits(), 0); + assert!(handle.snapshot_active.load(Ordering::SeqCst)); + let mut shutdown = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.shutdown().await } + }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut shutdown) + .await + .is_err() + ); + finish.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), shutdown) + .await + .unwrap() + .unwrap(); + assert_eq!(permits.available_permits(), 1); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); +} + #[tokio::test] async fn writer_snapshot_fence_rejects_input_even_when_the_callers_precheck_is_stale() { let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; @@ -679,6 +853,7 @@ async fn ordinary_restore_commit_cannot_consume_a_recovery_receipt_or_acknowledg confirmation: "a".repeat(64), original: origin, filesystem_complete: true, + resolution_digest: Some("d".repeat(64)), }); Ok(result) }) diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs index 2865af446..a4807d779 100644 --- a/crates/cyber-snapshot/src/copy/resolution.rs +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -227,7 +227,11 @@ impl Store { ) -> Result<()> { let origin = origin.ok_or("Recovery completion has no reserved native worker origin")?; let (_, journal) = self.journal(&origin.binding.journal_id)?; - if receipt(&origin.binding.journal_id, &journal)? != *expected { + let mut observed = receipt(&origin.binding.journal_id, &journal)?; + if expected.resolution_digest.is_none() { + observed.resolution_digest = None; + } + if observed != *expected { return Err("Native resolution receipt changed".into()); } Ok(()) @@ -258,10 +262,20 @@ impl Store { /// Called only after the runtime consumed the exact two-owner receipt. This /// deliberately checks immutable native proof, not later owner file edits. pub(super) fn acknowledge_resolution( + &self, + origin: &SnapshotOrigin, + dir: Dir, + journal: Journal, + ) -> Result<()> { + self.acknowledge_resolution_authorized(origin, dir, journal, &|| Ok(())) + } + + fn acknowledge_resolution_authorized( &self, origin: &SnapshotOrigin, dir: Dir, mut journal: Journal, + authorize: &dyn Fn() -> Result<()>, ) -> Result<()> { let resolution = journal .resolution @@ -282,12 +296,46 @@ impl Store { } // The child acknowledgement is durable before publishing the parent's // settlement link. A cut between writes stays fenced and is idempotent. + authorize()?; + self.verify_worker_namespace(origin)?; journal.acknowledged = true; Self::record(&dir, &journal)?; + authorize()?; + self.verify_worker_namespace(&resolution.original)?; parent.acknowledged = true; parent.settled_by = Some(origin.binding.journal_id.clone()); Self::record(&parent_dir, &parent) } + + pub(crate) fn reconcile_acknowledgement( + &self, + work: &cyber_server::runtime::SnapshotAcknowledgementWork<'_>, + ) -> Result<()> { + self.reconcile_receipt(work.origin(), work.receipt(), &|| work.authorize()) + } + pub(super) fn reconcile_receipt( + &self, + origin: &SnapshotOrigin, + expected: &Receipt, + authorize: &dyn Fn() -> Result<()>, + ) -> Result<()> { + authorize()?; + self.verify_worker_namespace(origin)?; + self.verify_receipt(Some(origin), expected)?; + let (dir, journal) = self.journal(&origin.binding.journal_id)?; + if expected.resolution_digest.is_none() { + let (_, parent) = self.journal(&expected.journal_id)?; + if journal.acknowledged + && parent.acknowledged + && parent.settled_by.as_deref() == Some(origin.binding.journal_id.as_str()) + { + self.verify_settlement(&parent)?; + return Ok(()); + } + return Err("Legacy receipt has no persisted immutable resolution digest; retained evidence cannot authorize new acknowledgement writes".into()); + } + self.acknowledge_resolution_authorized(origin, dir, journal, authorize) + } pub(super) fn resolution_preimages<'a>( &self, resolution: Option<&'a Resolution>, @@ -645,5 +693,6 @@ fn receipt(id: &str, record: &Journal) -> Result { confirmation: resolution.request.confirmation.clone(), original: resolution.original.clone(), filesystem_complete: true, + resolution_digest: Some(immutable_digest(record)?), }) } diff --git a/crates/cyber-snapshot/src/copy/resolution_tests.rs b/crates/cyber-snapshot/src/copy/resolution_tests.rs index a477416db..d803665a6 100644 --- a/crates/cyber-snapshot/src/copy/resolution_tests.rs +++ b/crates/cyber-snapshot/src/copy/resolution_tests.rs @@ -43,6 +43,111 @@ impl Prepared { .resolve_recovery(request, &|| Ok(()), None) .unwrap() } + fn resolution_with_origin( + &self, + request: &Confirmation, + ) -> ( + cyber_server::runtime::SnapshotRecoveryReceipt, + cyber_server::runtime::SnapshotOrigin, + ) { + let receipt = self.resolve(request); + let (dir, mut journal) = self.store.journal(&receipt.resolution_id).unwrap(); + let mut origin = self.record.origin.as_ref().unwrap().clone(); + origin.activity_id = "op_resolution_fixture".into(); + origin.binding.journal_id = receipt.resolution_id; + origin.binding.journal_identity = storage::dir_identity(&dir).unwrap(); + journal.origin = Some(origin.clone()); + Store::record(&dir, &journal).unwrap(); + (self.resolve(request), origin) + } +} + +#[test] +fn reconciliation_rejects_changed_intent_and_unverifiable_legacy_writes() { + let p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "reconcile-identity"); + let (receipt, origin) = p.resolution_with_origin(&request); + let mut legacy = receipt.clone(); + legacy.resolution_digest = None; + assert!( + p.store + .reconcile_receipt(&origin, &legacy, &|| Ok(())) + .is_err() + ); + let (dir, mut journal) = p.store.journal(&receipt.resolution_id).unwrap(); + journal.context = Some(super::super::RestoreContext { + target: format!("copy:{}", "0".repeat(64)), + recorded: format!("copy:{}", "0".repeat(64)), + expected: format!("copy:{}", "0".repeat(64)), + exclusions: vec!["tampered".into()], + }); + Store::record(&dir, &journal).unwrap(); + assert!( + p.store + .reconcile_receipt(&origin, &receipt, &|| Ok(())) + .is_err() + ); + assert!(!p.store.journal("restore_fixture").unwrap().1.acknowledged); + assert!( + !p.store + .journal(&receipt.resolution_id) + .unwrap() + .1 + .acknowledged + ); +} + +#[test] +fn reconciliation_cut_between_journal_writes_retains_fence_and_retry_preserves_owner_edits() { + let p = Prepared::managed(); + p.apply(0); + let request = p.confirmation(Action::Rollback, "reconcile-cut"); + let (receipt, origin) = p.resolution_with_origin(&request); + let calls = std::cell::Cell::new(0); + let stopped = || { + calls.set(calls.get() + 1); + if calls.get() == 3 { + Err("cancelled before parent acknowledgement".into()) + } else { + Ok(()) + } + }; + assert!( + p.store + .reconcile_receipt(&origin, &receipt, &stopped) + .is_err() + ); + assert!( + p.store + .journal(&receipt.resolution_id) + .unwrap() + .1 + .acknowledged + ); + assert!(!p.store.journal("restore_fixture").unwrap().1.acknowledged); + assert!(p.store.ensure_recovered().is_err()); + std::fs::write(p.root.join("existing/replace"), b"later owner data").unwrap(); + p.store + .reconcile_receipt(&origin, &receipt, &|| Ok(())) + .unwrap(); + p.store.ensure_recovered().unwrap(); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"later owner data" + ); + let (dir, _) = p.store.journal(&receipt.resolution_id).unwrap(); + let before = + storage::identity(&storage::open(&dir, Path::new("intent.json")).unwrap()).unwrap(); + let mut legacy = receipt; + legacy.resolution_digest = None; + p.store + .reconcile_receipt(&origin, &legacy, &|| Ok(())) + .unwrap(); + assert_eq!( + storage::identity(&storage::open(&dir, Path::new("intent.json")).unwrap()).unwrap(), + before + ); } #[tokio::test] @@ -59,6 +164,7 @@ async fn immutable_intent_handoff_rejects_changed_context_with_unchanged_files_a origin.binding.journal_identity = storage::dir_identity(&dir).unwrap(); journal.origin = Some(origin.clone()); Store::record(&dir, &journal).unwrap(); + let receipt = p.resolve(&request); let digest = p.store.worker_digest(Some(&origin)).unwrap(); let workspace = std::sync::Arc::new(tokio::sync::Mutex::new(())) .lock_owned() diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 97cec517f..f077be00a 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -315,6 +315,35 @@ impl Snapshots for GitSnapshots { } }) } + fn acknowledge_recovery( + &self, + directory: String, + worker: cyber_server::runtime::SnapshotAcknowledgementWorker, + ) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = (directory, worker); + Err("Native snapshot acknowledgement is unavailable on this host".into()) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + self.recovery_location(&directory)?; + let options = self.recovery_options(&directory); + let data = self.data_dir.clone(); + let lock = cyber_server::workspace::mutation_lock(Path::new(&directory)) + .map_err(|error| error.to_string())?; + let workspace = lock.lock_owned().await; + worker + .run(move |work| { + let _workspace = workspace; + let store = copy::Store::recover_existing(&data, &directory, options)?; + store.reconcile_acknowledgement(work) + }) + .await + } + }) + } fn owned_restore_available(&self, directory: &str) -> bool { cfg!(any(target_os = "macos", target_os = "linux")) && self.non_git(directory) } diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 4ac1d5ab5..96eabbf8c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -783,8 +783,9 @@ retry phases passed the repository threshold without allowances. Formatting and diff checks passed. macOS sandbox deprecation and Darwin linker warnings were observed without assigning their origin to main. -The complete feature is not merge-ready. Explicit journal acknowledgement -reconciliation after lost replies or failed flushes is still absent. Reservations +The complete feature is not merge-ready. At this checkpoint explicit journal +acknowledgement reconciliation after lost replies or failed flushes was absent; +the following continuation supplies it for committed recovery requests. Reservations that never publish a native origin remain unknown and lookup-only; missing records do not prove no effects or authorize clearing checkout/process uncertainty. Those resource/intent cuts require their own exact proof and operator resolution. Git @@ -794,3 +795,67 @@ cross-platform results and complete delivery review remain open. Linux/Windows, full-workspace checks and real SDK process acceptance were not run here. M2–M8 and both issues remain open. A remote refresh still reports main `db9e171e` and no open PRs; this increment has not been pushed or merged. + + +### Committed recovery-request acknowledgement reconciliation + +The explicit POST `/sessions/{id}/rewind/recovery/requests/{request_id}/acknowledge` +operation observes both durable native owners in one database read snapshot and +requires their exact committed receipt. It never admits an unknown or merely +filesystem-complete request. A separate runtime-issued metadata worker retains +actual lifecycle, Location and synchronous native store/workspace owners through +its last call. Its live control remains available to Stop; dropping the requester +cancels work but does not release those resources before the callback finishes. +The control is volatile because no new source effect or intent is admitted; the +already committed owner pair remains the durable recovery identity after a crash. + +New receipts persist an immutable resolution digest. Native reconciliation pins +that digest, the original intent and both private namespaces, then writes child +acknowledgement before the parent's settlement link, checking current runtime +authority before each write. A cut between writes retains the fence and supports +idempotent retry. No source-file scan, replacement, inference or transcript event +is involved. Older receipts without the digest can only observe an already fully +acknowledged settlement; they cannot grant new metadata writes. Unrelated opaque +uncertainty remains unchanged. + +Regression evidence includes a real file-backed Store committed-but-lost reply +followed by reconciliation despite its fenced writer, rejection of an uncommitted +filesystem receipt, and requester disposal while a synchronous callback retains a +real semaphore permit through shutdown. Native tests reject modified immutable +intent and missing legacy digest, cut between the two journal writes, then retry +while preserving later owner edits. The real CLI/API keep and rollback scenarios +also reset only mutable acknowledgement fields after a genuine prior partial- +restore kill, then acknowledge with a source replaced by an external-target symlink. +Both preserve that symlink, its target, transcript and inference count. This fixture +models the acknowledgement cut; it is not an actual process kill at that boundary. + +The recovery-request path does not yet reconcile an ordinary successful rewind's +Session/native receipt, reservations lacking native origin, Git journals, or +reclamation. Those remain required alongside complete clients and platform cuts. +M2–M8 and both issues remain open. + + +Verification for this continuation on local macOS: 406 server unit tests passed, +with the one configured subprocess fixture ignored; all 22 affected integration +cases passed (4 idle-cancellation, 13 Location, 5 rewind-preview). Snapshot tests +passed 72 cases (37 unit, 18 file-copy, 17 Git). The focused real CLI/API case +passed both keep and rollback scenarios. SDK typecheck and generation checks +passed; 131 SDK tests passed, with 10 configured process tests skipped because +`CYBER_BIN` was unset. OpenAPI consistency and final server/snapshot/CLI all-target +Clippy with warnings denied passed. Formatting, diff checks and strict OpenSpec +passed (117 items); spec lint reported 0 errors and 25 warnings, and inventory +remains 53 capabilities/1030 requirements. The initial integration invocation used +nonexistent shorthand target names; the corrected three actual targets above all +ran successfully. No Linux/Windows, full workspace, TUI/App recovery or real SDK +process acceptance was performed for this continuation. + +The code-review skill was applied to every changed runtime/native/HTTP/test and +generated SDK file. Review checked exact paired ownership and read-snapshot +consistency, digest compatibility, private namespace/lock binding, child-before- +parent durable ordering, cancellation/resource lifetime, preservation of generic +uncertainty, route ownership and replay separation. The review added the explicit +read transaction to the acknowledgement factory so both owners are observed from +one snapshot. No additional critical/major findings remain for this increment; +the missing broader recovery capabilities above remain acceptance gaps for the +complete feature. Architecture follows the repository's runtime port and native +adapter split. The increment remains local and does not make M2–M8 complete. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 9f7567aba..8b1a16ed6 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -189,3 +189,12 @@ count as positive native implementation acceptance. inactive branches, reader protection and cap pressure. - Linux x64/ARM and macOS Intel/ARM native positives and denials; explicit Windows results/limits; existing Git compatibility and shared binary recovery regressions. + + +Implementation note (2026-11-06): committed file-copy recovery requests now expose +an explicit metadata-only acknowledgement operation. New durable receipts bind an +immutable resolution digest; older receipts lacking it cannot authorize new journal +writes. The worker observes the exact committed owner pair, retains native locks, +and checks current authority before publishing child and parent acknowledgements. +Lookup remains read-only. Ordinary restore reconciliation, pre-origin cuts, Git +integration and reference-aware reclamation remain pending implementation. diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index f36ae2b6a..910d037c6 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -51,8 +51,11 @@ captured closed boundaries; ordinary native/hook admission is fenced at the writ Actual CLI/API service-kill cases resolve a partial namespace with keep and rollback, preserve unrelated/later owner files and the transcript, and repeat only lookup after restart. This is local non-Git evidence, not complete operator/client -or platform acceptance. Typed CLI/TUI recovery, explicit acknowledgement -reconciliation, pre-origin cuts and Git-native equivalents remain required. +or platform acceptance. Typed CLI/TUI recovery, ordinary-restore acknowledgement reconciliation, +pre-origin cuts and Git-native equivalents remain required. Explicit recovery-request +acknowledgement reconciliation now uses the exact committed pair and persisted +immutable resolution digest; it changes only journal metadata under retained native +locks and never rescans or reinstalls source files. Ordinary rewind commit still refuses recovery receipts. Filesystem completion alone never clears a native fence. Created source directories remain protected for safe acknowledged cleanup. @@ -66,9 +69,12 @@ uncertainty. Snapshot uncertainty is tracked by exact activity ID separately from the opaque flag; reload reconstructs generic fences and snapshot owners independently. A late-scope no-op clears only its exact snapshot reason after observing the matching durable acknowledgement at the writer. -Explicit reconciliation still needs to distinguish a durable writer +Explicit recovery-request reconciliation distinguishes a durable writer acknowledgement with missing journal acknowledgement from an uncommitted filesystem -outcome, and must not revalidate or replay later owner edits as the old request. +outcome. Regression coverage refuses the latter before calling the native adapter, +and preserves later owner edits, including a source replaced with a symlink. +Older receipts without an immutable resolution digest cannot authorize new journal +writes. Ordinary restore receipts still require their own reconciliation path. Unprepared reservation/origin/intent cuts still require exact resource ownership proof; missing records alone do not license acknowledgement. @@ -82,7 +88,7 @@ The non-Git reviewed-confirm path now uses a runtime-issued synchronous worker, origin-bound intent and paired Session/native commit followed by journal acknowledgement. The recovery path adds globally reserved request ownership and paired original/resolution settlement. M2–M8 remain open: Git and legacy entry-point -integration, pre-origin resource proof, lost-reply journal reconciliation, cleanup, +integration, pre-origin resource proof, ordinary-restore journal reconciliation, cleanup, complete clients and native/platform acceptance are still required. New version-3 file-copy intents retain native per-change source/staged file and diff --git a/sdk/openapi.json b/sdk/openapi.json index e13f3b1c5..bdef31eec 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7466,6 +7466,13 @@ }, "filesystem_complete": { "type": "boolean" + }, + "resolution_digest": { + "type": [ + "string", + "null" + ], + "description": "Immutable native resolution data, excluding acknowledgement/progress.\nMissing on older receipts; those cannot authorize new journal writes." } }, "required": [ @@ -24505,6 +24512,63 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}/acknowledge": { + "post": { + "operationId": "v1.session.snapshotRecoveryAcknowledge", + "summary": "Reconcile journal acknowledgement for the exact committed native pair without replaying source changes or starting inference", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "requestID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotRecoveryResult" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/preview": { "post": { "operationId": "v1.session.rewindPreview", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index b342b1d76..81e61f4b7 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1559,6 +1559,12 @@ export interface Operations { body: undefined; response: S.SnapshotRecovery[]; }; + "v1.session.snapshotRecoveryAcknowledge": { + path: { sessionID: string; requestID: string }; + query: Record; + body: undefined; + response: S.SnapshotRecoveryResult; + }; "v1.session.snapshotRecoveryConfirm": { path: { sessionID: string }; query: Record; @@ -4662,6 +4668,18 @@ export const operations = { unwrap: true, stream: false, }, + "v1.session.snapshotRecoveryAcknowledge": { + tag: "session", + name: "snapshotRecoveryAcknowledge", + method: "POST", + path: "/api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}/acknowledge", + pathParams: ["sessionID", "requestID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, "v1.session.snapshotRecoveryConfirm": { tag: "session", name: "snapshotRecoveryConfirm", @@ -6091,6 +6109,9 @@ export function createGroups(caller: Caller) { /** Inspect retained filesystem restore journals without acknowledging or replaying native effects (`GET /api/v1/sessions/{sessionID}/rewind/recovery`) */ snapshotRecovery: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecovery", { path: { sessionID } }, options), + /** Reconcile journal acknowledgement for the exact committed native pair without replaying source changes or starting inference (`POST /api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}/acknowledge`) */ + snapshotRecoveryAcknowledge: (sessionID: string, requestID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotRecoveryAcknowledge", { path: { sessionID, requestID } }, options), /** Resolve fingerprint-confirmed native snapshot effects with an exact durable request; repeated IDs are lookup-only (`POST /api/v1/sessions/{sessionID}/rewind/recovery/confirm`) */ snapshotRecoveryConfirm: (sessionID: string, body: Operations["v1.session.snapshotRecoveryConfirm"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecoveryConfirm", { path: { sessionID }, body }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 551f67bdb..ef1a23bf8 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -4762,6 +4762,11 @@ export type SnapshotRecoveryReceipt = { confirmation: string; original: SnapshotOrigin; filesystem_complete: boolean; + /** + * Immutable native resolution data, excluding acknowledgement/progress. + * Missing on older receipts; those cannot authorize new journal writes. + */ + resolution_digest?: string | null; }; export type SnapshotRecoveryResult = { diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index 86b4e8f40..becd7b846 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -46,3 +46,15 @@ test("snapshot recovery confirmation and lookup retain the exact request without assert.equal(calls[1]!.method, "GET"); assert.equal(calls[1]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/requests/request%2Fwith%20space"); }); + +test("snapshot recovery acknowledgement uses the retained request without a replay body", async () => { + const result = { state: "acknowledged", journal_acknowledged: true, receipt: { resolution_digest: "d".repeat(64) } }; + const { client, calls } = mockClient(() => json(200, { data: result })); + const acknowledged = await client.session.snapshotRecoveryAcknowledge("ses_owner", "request/with space"); + assert.equal(acknowledged.journal_acknowledged, true); + assert.equal(acknowledged.receipt?.resolution_digest, "d".repeat(64)); + assert.equal(calls.length, 1); + assert.equal(calls[0]!.method, "POST"); + assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/requests/request%2Fwith%20space/acknowledge"); + assert.ok(!calls[0]!.body); +}); From 36c6b1aa953845eb681d6a4374652d8e900a9ed7 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 20:42:42 -0400 Subject: [PATCH 09/87] Reconcile ordinary restore acknowledgements from paired receipts --- crates/cyber-cli/tests/branches_cli.rs | 45 ++++ crates/cyber-server/src/http/openapi.rs | 4 + crates/cyber-server/src/http/sessions.rs | 36 ++++ crates/cyber-server/src/runtime/activity.rs | 15 ++ crates/cyber-server/src/runtime/mod.rs | 3 +- .../src/runtime/snapshot_acknowledgement.rs | 56 ++++- .../src/runtime/snapshot_restore_receipt.rs | 174 ++++++++++++++++ .../cyber-server/src/runtime/snapshot_work.rs | 32 ++- .../src/runtime/snapshot_work_tests.rs | 193 ++++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 3 + crates/cyber-snapshot/src/copy/resolution.rs | 29 ++- .../src/copy/resolution_tests.rs | 66 ++++++ .../implementation.md | 76 ++++++- .../managed-recovery.md | 18 +- .../specs/snapshots-checkpoints/spec.md | 11 + .../complete-daily-coding-surfaces/tasks.md | 10 +- sdk/openapi.json | 148 ++++++++++++++ sdk/typescript/src/generated/operations.ts | 42 ++++ sdk/typescript/src/generated/types.ts | 16 ++ sdk/typescript/test/rewind.test.ts | 13 ++ 20 files changed, 973 insertions(+), 17 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_restore_receipt.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 8af2abae5..d665b02f6 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1099,6 +1099,51 @@ fn reviewed_non_git_code_rewind_uses_private_snapshots_and_keeps_owner_files() { assert_eq!(model.requests().len(), requests); let all = json(&env.cyber(&["sessions", "export", &id, "--all-branches"])); assert!(all.to_string().contains(&first)); + reconcile_ordinary_restore_metadata(&env, &id, journals[0]["id"].as_str().unwrap()); + assert_eq!(model.requests().len(), requests); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn reconcile_ordinary_restore_metadata(env: &Env, session: &str, journal: &str) { + let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let path = namespace.join("journals").join(journal).join("intent.json"); + let mut intent: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let activity = intent["origin"]["activity_id"].as_str().unwrap().to_owned(); + let route = format!("/sessions/{session}/rewind/restores/{activity}"); + let receipt = json(&env.cyber(&["api", "GET", &route]))["data"].clone(); + assert_eq!(receipt["origin"], intent["origin"]); + assert_eq!(receipt["intent_digest"].as_str().unwrap().len(), 64); + // Simulate an acknowledgement flush cut without changing immutable intent. + intent["acknowledged"] = false.into(); + std::fs::write(&path, serde_json::to_vec(&intent).unwrap()).unwrap(); + let target = env.root.join("ordinary-ack-owner"); + std::fs::write(&target, "later owner bytes").unwrap(); + let source = env.root.join("repo/a.txt"); + std::os::unix::fs::symlink(&target, &source).unwrap(); + let before = env.messages(session); + for _ in 0..2 { + let result = + json(&env.cyber(&["api", "POST", &format!("{route}/acknowledge"), "-d", "{}"])); + assert_eq!(result["data"], receipt); + } + let acknowledged: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + assert_eq!(acknowledged["acknowledged"], true); + assert!( + std::fs::symlink_metadata(source) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_to_string(target).unwrap(), + "later owner bytes" + ); + assert_eq!(env.messages(session), before); } #[cfg(any(target_os = "macos", target_os = "linux"))] diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index 2d8593c92..a566cdb61 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -770,6 +770,10 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { .ok::>(g), op("post", "/sessions/{sessionID}/rewind/recovery/requests/{requestID}/acknowledge", "v1.session.snapshotRecoveryAcknowledge", "Reconcile journal acknowledgement for the exact committed native pair without replaying source changes or starting inference") .ok::>(g), + op("get", "/sessions/{sessionID}/rewind/restores/{activityID}", "v1.session.snapshotRestoreLookup", "Read the exact committed ordinary restore receipt without native access or replay") + .ok::>(g), + op("post", "/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge", "v1.session.snapshotRestoreAcknowledge", "Acknowledge the immutable ordinary restore intent after its paired Session commit without replaying source files") + .ok::>(g), op("post", "/sessions/{sessionID}/rewind/preview", "v1.session.rewindPreview", "Review a rewind without changing files or the active conversation") .body::(g) .ok::>(g), diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index 19279e39d..beb737459 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -362,6 +362,14 @@ pub fn routes() -> Router { &format!("{s}/rewind/recovery/requests/{{request_id}}/acknowledge"), post(snapshot_recovery_acknowledge), ) + .route( + &format!("{s}/rewind/restores/{{activity_id}}"), + get(snapshot_restore_lookup), + ) + .route( + &format!("{s}/rewind/restores/{{activity_id}}/acknowledge"), + post(snapshot_restore_acknowledge), + ) .route(&format!("{s}/rewind/confirm"), post(rewind_confirm)) .route(&format!("{s}/revert/stage"), post(stage)) .route(&format!("{s}/revert/clear"), post(clear)) @@ -1265,6 +1273,34 @@ async fn snapshot_recovery_acknowledge( .await?, })) } +async fn snapshot_restore_lookup( + State(state): State, + Path((id, activity_id)): Path<(String, String)>, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_restore_lookup(&id, &activity_id) + .await? + .ok_or_else(|| { + ApiError::not_found( + "RestoreNotFoundError", + "No immutable committed restore receipt", + ) + })?, + })) +} +async fn snapshot_restore_acknowledge( + State(state): State, + Path((id, activity_id)): Path<(String, String)>, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_restore_acknowledge(&id, &activity_id) + .await?, + })) +} async fn rewind_preview( State(state): State, Path(id): Path, diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index c6ea82efe..7f1197395 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -34,6 +34,8 @@ pub(super) struct Record { #[serde(default, skip_serializing_if = "Option::is_none")] pub snapshot_receipt: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_restore_receipt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_bindings: Option>, } pub(super) struct Scope { @@ -117,6 +119,7 @@ impl Scope { snapshot_origin: None, snapshot_request: Some(request.clone()), snapshot_receipt: None, + snapshot_restore_receipt: None, admission_bindings: Some(authority.bindings().to_vec()), }; let captured = authority.clone(); @@ -181,6 +184,7 @@ impl Scope { snapshot_origin: None, snapshot_request: None, snapshot_receipt: None, + snapshot_restore_receipt: None, admission_bindings: Some(authority.bindings.clone()), }; inner @@ -253,6 +257,7 @@ impl Scope { state: &mut super::SessionState, sequence: i64, events: Vec, + intent_digest: Option<&str>, ) -> Result<(), RuntimeError> { let runtime = Runtime { inner: self.inner.upgrade().ok_or(RuntimeError::ShuttingDown)?, @@ -272,6 +277,16 @@ impl Scope { settled.status = "settled".into(); settled.phase = "snapshot_restore_committed".into(); settled.admission_bindings = None; + settled.snapshot_restore_receipt = intent_digest + .map(|digest| { + super::snapshot_restore_receipt::capture( + self.snapshot_origin(), + digest, + sequence, + &events, + ) + }) + .transpose()?; let bindings = authority.bindings.clone(); let acknowledgement = change(&settled); let owners = [ diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index b8b1c19d6..cd0392e62 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -129,6 +129,7 @@ mod rewind; mod rewind_preview; mod snapshot_recovery; mod snapshot_recovery_types; +mod snapshot_restore_receipt; mod snapshot_work; pub use rewind_preview::{ RewindAction, RewindCodePreview, RewindConfirmation, RewindEffect, RewindPreview, @@ -141,7 +142,7 @@ pub use snapshot_recovery_types::{ }; pub use snapshot_work::{ SnapshotBinding, SnapshotCompletion, SnapshotNativeGuard, SnapshotNativeResult, SnapshotOrigin, - SnapshotRestoreRequest, SnapshotWork, SnapshotWorker, + SnapshotRestoreReceipt, SnapshotRestoreRequest, SnapshotWork, SnapshotWorker, }; mod routing; mod sandbox_events; diff --git a/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs index 257f56def..73fa11c8f 100644 --- a/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs +++ b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs @@ -10,7 +10,7 @@ pub struct SnapshotAcknowledgementWorker { runtime: Runtime, authority: RecoveryAuthority, sequence: i64, - receipt: SnapshotRecoveryReceipt, + receipt: Acknowledgement, origin: SnapshotOrigin, control: Arc, // Actual resources are released before ordinary admission or Stop waiters. @@ -19,6 +19,28 @@ pub struct SnapshotAcknowledgementWorker { _admission: Admission, _completion: Completion, } +#[derive(Clone)] +pub(super) enum Acknowledgement { + Recovery(SnapshotRecoveryReceipt), + Restore(super::SnapshotRestoreReceipt), +} +impl Acknowledgement { + fn verify( + &self, + db: &rusqlite::Connection, + origin: &SnapshotOrigin, + ) -> Result<(), cyber_store::StoreError> { + match self { + Self::Recovery(receipt) => verify_pair(db, receipt, origin), + Self::Restore(receipt) if receipt.origin == *origin => { + super::snapshot_restore_receipt::verify(db, receipt) + } + Self::Restore(_) => Err(cyber_store::StoreError::InvalidAppend( + "Foreign restore origin".into(), + )), + } + } +} struct Admission(Arc); impl Drop for Admission { fn drop(&mut self) { @@ -46,8 +68,17 @@ impl Drop for Completion { /// installation, intent creation, generic launch authority or worker-origin adoption. pub struct SnapshotAcknowledgementWork<'a>(&'a SnapshotAcknowledgementWorker); impl SnapshotAcknowledgementWork<'_> { - pub fn receipt(&self) -> &SnapshotRecoveryReceipt { - &self.0.receipt + pub fn recovery_receipt(&self) -> Option<&SnapshotRecoveryReceipt> { + match &self.0.receipt { + Acknowledgement::Recovery(receipt) => Some(receipt), + _ => None, + } + } + pub fn restore_receipt(&self) -> Option<&super::SnapshotRestoreReceipt> { + match &self.0.receipt { + Acknowledgement::Restore(receipt) => Some(receipt), + _ => None, + } } pub fn origin(&self) -> &SnapshotOrigin { &self.0.origin @@ -66,7 +97,7 @@ impl SnapshotAcknowledgementWork<'_> { .store .read(move |db| { let tx = db.unchecked_transaction()?; - verify_pair(&tx, &receipt, &origin)?; + receipt.verify(&tx, &origin)?; let current: i64 = tx.query_row( "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", [&origin.session_id], @@ -107,6 +138,21 @@ impl Runtime { state: &SessionState, receipt: SnapshotRecoveryReceipt, origin: SnapshotOrigin, + ) -> Result { + self.snapshot_acknowledgement_worker_for( + handle, + state, + Acknowledgement::Recovery(receipt), + origin, + ) + .await + } + pub(super) async fn snapshot_acknowledgement_worker_for( + &self, + handle: &Arc, + state: &SessionState, + receipt: Acknowledgement, + origin: SnapshotOrigin, ) -> Result { let lifecycle = self.memory_mutation_lease().await?; self.inner.ensure_idle(&state.info.id)?; @@ -137,7 +183,7 @@ impl Runtime { let bound_origin = origin.clone(); self.inner.store.read(move |db| { let tx = db.unchecked_transaction()?; - verify_pair(&tx, &bound_receipt, &bound_origin)?; + bound_receipt.verify(&tx, &bound_origin)?; tx.commit()?; Ok(()) })?; diff --git a/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs new file mode 100644 index 000000000..c7af8f1de --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs @@ -0,0 +1,174 @@ +//! Immutable ordinary-restore settlement evidence, independent of later owner files. +use super::{Runtime, RuntimeError, SnapshotOrigin, SnapshotRestoreReceipt, snapshot_work}; +use cyber_store::{NewEvent, StoreError}; +use rusqlite::Connection; + +const MAX_EVENTS: i64 = 4096; + +impl Runtime { + /// Lookup only; legacy settlements without an immutable receipt return None. + pub async fn snapshot_restore_lookup( + &self, + session: &str, + activity: &str, + ) -> Result, RuntimeError> { + use rusqlite::OptionalExtension; + let source = session.to_owned(); + let owner = activity.to_owned(); + self.inner.store.read(move |db| { + let tx = db.unchecked_transaction()?; + let record = tx.query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq DESC LIMIT 1", + [&owner, super::activity::CHANGED], |row| row.get::<_, String>(0), + ).optional()?; + let receipt = record.map(|data| serde_json::from_str::(&data) + .map_err(|error| StoreError::InvalidAppend(error.to_string()))) + .transpose()?.filter(|record| record.session_id == source) + .and_then(|record| record.snapshot_restore_receipt); + if let Some(receipt) = &receipt { + if receipt.origin.session_id != source || receipt.origin.activity_id != owner { + return Err(StoreError::InvalidAppend("Foreign restore receipt owner".into())); + } + verify(&tx, receipt)?; + } + tx.commit()?; + Ok(receipt) + }).map_err(RuntimeError::from) + } + + /// Acknowledge an exact already committed ordinary restore; never replay it. + pub async fn snapshot_restore_acknowledge( + &self, + session: &str, + activity: &str, + ) -> Result { + let receipt = self + .snapshot_restore_lookup(session, activity) + .await? + .ok_or_else(|| { + RuntimeError::Conflict("No immutable committed restore receipt".into()) + })?; + let handle = self.inner.handle(session).await?; + let state = handle.state.lock().await; + let directory = state.info.directory.clone(); + let worker = self + .snapshot_acknowledgement_worker_for( + &handle, + &state, + super::snapshot_acknowledgement::Acknowledgement::Restore(receipt.clone()), + receipt.origin.clone(), + ) + .await?; + drop(state); + self.inner + .options + .snapshots + .acknowledge_recovery(directory, worker) + .await + .map_err(RuntimeError::Conflict)?; + handle.acknowledge_snapshot_uncertainty(&receipt.origin.activity_id); + Ok(receipt) + } +} + +pub(super) fn capture( + origin: Option<&SnapshotOrigin>, + digest: &str, + sequence: i64, + events: &[NewEvent], +) -> Result { + let origin = origin.ok_or_else(|| { + RuntimeError::Conflict("Immutable snapshot receipt lacks its native origin".into()) + })?; + let count = i64::try_from(events.len()) + .map_err(|_| RuntimeError::Invalid("Snapshot event batch is too large".into()))?; + if digest.len() != 64 + || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + || sequence < 0 + || count > MAX_EVENTS + { + return Err(RuntimeError::Conflict( + "Invalid immutable snapshot receipt".into(), + )); + } + Ok(SnapshotRestoreReceipt { + origin: origin.clone(), + intent_digest: digest.to_owned(), + session_before: sequence, + session_after: sequence + .checked_add(count) + .ok_or_else(|| RuntimeError::Invalid("Snapshot Session sequence overflow".into()))?, + events_digest: snapshot_work::restore_events_digest(events) + .map_err(RuntimeError::Invalid)?, + }) +} + +/// Caller must hold one database read transaction across the complete proof. +pub(super) fn verify(db: &Connection, receipt: &SnapshotRestoreReceipt) -> Result<(), StoreError> { + let (_, native) = super::snapshot_recovery_record::attest(db, &receipt.origin)?; + let invalid = || StoreError::InvalidAppend("Ordinary restore settlement proof differs".into()); + if native.status != "settled" + || native.phase != "snapshot_restore_committed" + || native.snapshot_restore_receipt.as_ref() != Some(receipt) + || native.snapshot_request.is_some() + || native.snapshot_receipt.is_some() + || receipt.session_before < 0 + || receipt.session_after < receipt.session_before + || receipt.session_after - receipt.session_before > MAX_EVENTS + { + return Err(invalid()); + } + let created: String = db.query_row( + "SELECT id FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq LIMIT 1", + [&receipt.origin.session_id, super::events::CREATED], + |row| row.get(0), + )?; + if created != receipt.origin.session_created_id { + return Err(invalid()); + } + let anchor: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM event WHERE aggregate_id=?1 AND seq=?2)", + rusqlite::params![receipt.origin.session_id, receipt.session_before], + |row| row.get(0), + )?; + if !anchor { + return Err(invalid()); + } + let mut query = db.prepare( + "SELECT seq,type,data,causation_id FROM event WHERE aggregate_id=?1 AND seq>?2 AND seq<=?3 ORDER BY seq", + )?; + let rows = query.query_map( + rusqlite::params![ + receipt.origin.session_id, + receipt.session_before, + receipt.session_after + ], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + }, + )?; + let mut events = Vec::new(); + for row in rows { + let (seq, kind, data, causation_id) = row?; + if seq != receipt.session_before + events.len() as i64 + 1 { + return Err(invalid()); + } + events.push(NewEvent { + kind, + data: serde_json::from_str(&data).map_err(|_| invalid())?, + causation_id, + }); + } + if receipt.session_after - receipt.session_before != events.len() as i64 + || snapshot_work::restore_events_digest(&events).map_err(|_| invalid())? + != receipt.events_digest + { + return Err(invalid()); + } + Ok(()) +} diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 812132824..e43a55e25 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -33,6 +33,24 @@ pub struct SnapshotOrigin { pub binding: SnapshotBinding, } +/// Exact native intent and Session batch consumed by an ordinary restore. +/// This is durable acknowledgement evidence, never authority to replay files. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotRestoreReceipt { + pub origin: SnapshotOrigin, + pub intent_digest: String, + pub session_before: i64, + pub session_after: i64, + pub events_digest: String, +} + +pub(super) fn restore_events_digest(events: &[NewEvent]) -> Result { + use sha2::{Digest, Sha256}; + let bytes = serde_json::to_vec(events).map_err(|error| error.to_string())?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} + pub struct SnapshotRestoreRequest { pub directory: String, pub target: String, @@ -43,6 +61,11 @@ pub struct SnapshotRestoreRequest { /// Native locks and descriptors held through the consuming runtime commit. pub trait SnapshotNativeGuard: Send { + /// Captured immutable intent under the retained native locks. Older adapters + /// may omit this; their settlement cannot authorize later journal writes. + fn intent_digest(&self) -> Option<&str> { + None + } fn verify(&self, origin: Option<&SnapshotOrigin>) -> Result<(), String>; /// Called only after observed durable runtime acknowledgement, under retained locks. fn acknowledge(&mut self, origin: Option<&SnapshotOrigin>) -> Result<(), String>; @@ -254,9 +277,12 @@ impl SnapshotCompletion { .guard .verify(self.worker.scope.snapshot_origin()) .map_err(RuntimeError::Conflict)?; - self.worker - .scope - .commit_snapshot(state, self.worker.sequence, events)?; + self.worker.scope.commit_snapshot( + state, + self.worker.sequence, + events, + self.native.guard.intent_digest(), + )?; self.native .guard .acknowledge(self.worker.scope.snapshot_origin()) diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 3ca1c00c6..82ebc7565 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -9,6 +9,9 @@ struct Guard { acknowledgements: Arc, } impl SnapshotNativeGuard for Guard { + fn intent_digest(&self) -> Option<&str> { + Some("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd") + } fn verify(&self, origin: Option<&SnapshotOrigin>) -> Result<(), String> { if self.origin.as_ref() != origin { return Err("foreign worker receipt".into()); @@ -135,6 +138,150 @@ impl Snapshots for Acknowledgements { } } +#[tokio::test] +async fn ordinary_restore_lost_pair_reply_retains_exact_receipt_without_replaying_the_worker() { + fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("lost ordinary restore reply".into()), + )) + } + let (temp, base, _, _, _) = super::super::binary_documents_tests::fixture().await; + let path = temp.path().join("ordinary-owner.db"); + let store = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(path.clone()), + Runtime::registry(), + )) + .unwrap(); + let acknowledged = Arc::new(AtomicUsize::new(0)); + let runtime = runtime_with_snapshots( + &base, + Arc::new(store), + temp.path(), + Arc::new(Acknowledgements(acknowledged.clone())), + ); + let session = runtime + .create_session(super::super::CreateSession { + directory: temp.path().to_str().unwrap().into(), + model: "test/main".into(), + ..Default::default() + }) + .await + .unwrap() + .id; + let source_ack = Arc::new(AtomicUsize::new(0)); + let count = source_ack.clone(); + let binding = binding(temp.path().canonicalize().unwrap()); + let completed = worker(&runtime, &session) + .await + .run(move |work| { + let origin = work.begin(binding).map_err(RestoreError::Failed)?; + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: count, + }), + )) + }) + .await + .unwrap(); + let activity = completed.worker.scope.id().to_owned(); + runtime + .inner + .store + .transaction(|db| { + db.wal_hook(Some(lost_reply)); + Ok(()) + }) + .unwrap(); + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + let before = state.last_seq; + let notifications = runtime.inner.store.subscribe(); + assert!( + completed + .commit( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "durable restore batch".into(), + usage: None, + cost: None, + model: None + } + )] + ) + .is_err() + ); + drop(state); + assert!(notifications.try_recv().is_err()); + let receipt = runtime + .snapshot_restore_lookup(&session, &activity) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.session_before, before); + assert_eq!(receipt.session_after, before + 1); + assert_eq!(source_ack.load(Ordering::SeqCst), 0); + assert!( + runtime + .snapshot_restore_acknowledge(&session, &activity) + .await + .is_err() + ); + assert_eq!(acknowledged.load(Ordering::SeqCst), 0); + runtime.shutdown().await; + drop(handle); + drop(runtime); + let reopened = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(path.clone()), + Runtime::registry(), + )) + .unwrap(); + let reloaded = runtime_with_snapshots( + &base, + Arc::new(reopened), + temp.path(), + Arc::new(Acknowledgements(acknowledged.clone())), + ); + assert_eq!( + reloaded + .snapshot_restore_lookup(&session, &activity) + .await + .unwrap(), + Some(receipt.clone()) + ); + let handle = reloaded.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + reloaded + .inner + .commit_locked( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "later owner revision".into(), + usage: None, + cost: None, + model: None, + }, + )], + ) + .unwrap(); + drop(state); + let reconciled = reloaded + .snapshot_restore_acknowledge(&session, &activity) + .await + .unwrap(); + assert_eq!(reconciled, receipt); + assert_eq!(acknowledged.load(Ordering::SeqCst), 1); + assert_eq!(source_ack.load(Ordering::SeqCst), 0); + reloaded.shutdown().await; + base.shutdown().await; +} + #[tokio::test] async fn recovery_pair_lost_commit_reply_is_lookup_only_after_reopening_the_real_store() { fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { @@ -335,6 +482,19 @@ async fn acknowledgement_cannot_adopt_an_uncommitted_filesystem_receipt() { Arc::new(Acknowledgements(count.clone())), ); let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + assert!( + runtime + .snapshot_restore_lookup(&session, &original.activity_id) + .await + .unwrap() + .is_none() + ); + assert!( + runtime + .snapshot_restore_acknowledge(&session, &original.activity_id) + .await + .is_err() + ); let request = recovery_request(original); let handle = runtime.inner.handle(&session).await.unwrap(); let state = handle.state.lock().await; @@ -1007,6 +1167,39 @@ async fn snapshot_completion_commits_session_and_exact_origin_before_native_ack( .id .clone(); assert_eq!(record["snapshot_origin"]["session_created_id"], created); + let activity = record["snapshot_origin"]["activity_id"].as_str().unwrap(); + let receipt = runtime + .snapshot_restore_lookup(&session, activity) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.origin.activity_id, activity); + assert_eq!(receipt.session_after - receipt.session_before, 1); + assert!( + runtime + .snapshot_restore_lookup("ses_foreign", activity) + .await + .unwrap() + .is_none() + ); + let mut changed = receipt.clone(); + changed.events_digest = "0".repeat(64); + assert!( + runtime + .inner + .store + .read(move |db| { super::super::snapshot_restore_receipt::verify(db, &changed) }) + .is_err() + ); + let mut changed = receipt; + changed.intent_digest = "0".repeat(64); + assert!( + runtime + .inner + .store + .read(move |db| { super::super::snapshot_restore_receipt::verify(db, &changed) }) + .is_err() + ); assert!(record.get("binary_document").is_none()); assert_eq!(acknowledgements.load(Ordering::SeqCst), 1); let unrelated = runtime diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index c0d4cf245..702f85eb3 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -77,6 +77,9 @@ pub(crate) struct WorkerGuard { } impl cyber_server::runtime::SnapshotNativeGuard for WorkerGuard { + fn intent_digest(&self) -> Option<&str> { + self.digest.as_deref() + } fn verify(&self, origin: Option<&cyber_server::runtime::SnapshotOrigin>) -> Result<()> { let _retained_workspace = &self.workspace; if origin != self.origin.as_ref() { diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs index a4807d779..347460a25 100644 --- a/crates/cyber-snapshot/src/copy/resolution.rs +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -311,7 +311,34 @@ impl Store { &self, work: &cyber_server::runtime::SnapshotAcknowledgementWork<'_>, ) -> Result<()> { - self.reconcile_receipt(work.origin(), work.receipt(), &|| work.authorize()) + if let Some(receipt) = work.recovery_receipt() { + return self.reconcile_receipt(work.origin(), receipt, &|| work.authorize()); + } + let receipt = work + .restore_receipt() + .ok_or("Missing acknowledgement receipt")?; + self.reconcile_restore_receipt(receipt, &|| work.authorize()) + } + pub(super) fn reconcile_restore_receipt( + &self, + receipt: &cyber_server::runtime::SnapshotRestoreReceipt, + authorize: &dyn Fn() -> Result<()>, + ) -> Result<()> { + let origin = &receipt.origin; + authorize()?; + self.verify_worker_namespace(origin)?; + let (dir, mut journal) = self.journal(&origin.binding.journal_id)?; + if journal.origin.as_ref() != Some(origin) + || !journal.completed + || journal.resolution.is_some() + || immutable_digest(&journal)? != receipt.intent_digest + { + return Err("Ordinary restore intent differs from its committed receipt".into()); + } + authorize()?; + self.verify_worker_namespace(origin)?; + journal.acknowledged = true; + Self::record(&dir, &journal) } pub(super) fn reconcile_receipt( &self, diff --git a/crates/cyber-snapshot/src/copy/resolution_tests.rs b/crates/cyber-snapshot/src/copy/resolution_tests.rs index d803665a6..0b9bb0f30 100644 --- a/crates/cyber-snapshot/src/copy/resolution_tests.rs +++ b/crates/cyber-snapshot/src/copy/resolution_tests.rs @@ -62,6 +62,72 @@ impl Prepared { } } +#[test] +fn ordinary_reconciliation_requires_completed_immutable_intent_and_current_authority() { + let p = Prepared::managed(); + let mut journal = p.record.clone(); + let mut receipt = cyber_server::runtime::SnapshotRestoreReceipt { + origin: journal.origin.clone().unwrap(), + intent_digest: p + .store + .worker_digest(journal.origin.as_ref()) + .unwrap() + .unwrap(), + session_before: 0, + session_after: 1, + events_digest: "a".repeat(64), + }; + assert!( + p.store + .reconcile_restore_receipt(&receipt, &|| Ok(())) + .is_err() + ); + for index in 0..journal.changes.len() { + p.apply(index); + } + journal.applied = journal + .changes + .iter() + .map(|change| change.path.clone()) + .collect(); + journal.completed = true; + Store::record(&p.dir, &journal).unwrap(); + receipt.intent_digest = "0".repeat(64); + assert!( + p.store + .reconcile_restore_receipt(&receipt, &|| Ok(())) + .is_err() + ); + receipt.intent_digest = p + .store + .worker_digest(journal.origin.as_ref()) + .unwrap() + .unwrap(); + let calls = std::cell::Cell::new(0); + assert!( + p.store + .reconcile_restore_receipt(&receipt, &|| { + calls.set(calls.get() + 1); + if calls.get() == 2 { + Err("revoked before metadata write".into()) + } else { + Ok(()) + } + }) + .is_err() + ); + assert!(!p.store.journal("restore_fixture").unwrap().1.acknowledged); + std::fs::write(p.root.join("existing/replace"), b"later owner bytes").unwrap(); + p.store + .reconcile_restore_receipt(&receipt, &|| Ok(())) + .unwrap(); + assert!(p.store.journal("restore_fixture").unwrap().1.acknowledged); + assert_eq!( + std::fs::read(p.root.join("existing/replace")).unwrap(), + b"later owner bytes" + ); +} + #[test] fn reconciliation_rejects_changed_intent_and_unverifiable_legacy_writes() { let p = Prepared::managed(); diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 96eabbf8c..417e5a774 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -829,7 +829,7 @@ restore kill, then acknowledge with a source replaced by an external-target syml Both preserve that symlink, its target, transcript and inference count. This fixture models the acknowledgement cut; it is not an actual process kill at that boundary. -The recovery-request path does not yet reconcile an ordinary successful rewind's +At that checkpoint, the recovery-request path did not reconcile an ordinary successful rewind's Session/native receipt, reservations lacking native origin, Git journals, or reclamation. Those remain required alongside complete clients and platform cuts. M2–M8 and both issues remain open. @@ -859,3 +859,77 @@ one snapshot. No additional critical/major findings remain for this increment; the missing broader recovery capabilities above remain acceptance gaps for the complete feature. Architecture follows the repository's runtime port and native adapter split. The increment remains local and does not make M2–M8 complete. + + +### Ordinary file-copy restore acknowledgement reconciliation + +New ordinary restore settlements retain `SnapshotRestoreReceipt` on their exact +native activity. The native guard supplies its captured immutable intent digest; +the existing paired append binds it to the same Session event batch by sequence +range and SHA-256 digest of serialized event kind/data/causation IDs. The batch is +bounded to 4096 events. No new Session event or transcript movement is introduced. +Older adapters omitting immutable proof retain their existing settlement behavior, +but cannot authorize later acknowledgement writes through this API. + +GET `/sessions/{id}/rewind/restores/{activity_id}` observes the original native +settlement and exact Session batch in a single read transaction; absent, foreign +or historical receipts without this proof return 404. POST to its `/acknowledge` +child obtains the same metadata-only worker resources used for recovery requests. +The worker verifies current Session revision/authority before the native adapter +checks original lock, private namespace and journal identity, completed ordinary +intent and immutable digest. It writes only the acknowledgement flag. It neither +rescans source files nor invokes restore. Later Session batches do not invalidate +the original receipt, but changes during the acquired worker remain fenced. + +The real CLI/API regression resets only mutable journal acknowledgement after a +successful ordinary non-Git rewind, creates a symlink at the restored path to an +external file, then performs acknowledgement twice. Both responses retain exactly +the lookup receipt, and the source symlink, target bytes, transcript and inference +count remain unchanged. This is a simulated acknowledgement-flush cut, not a real +process kill at that boundary. Additional runtime coverage uses an actual file- +backed Store WAL hook to lose the paired commit reply, retains lookup evidence, +refuses acknowledgement against stale in-memory Session state, reopens the Store, +adds a later Session revision, and reconciles without calling the source worker. +The native adapter fixture refuses incomplete/digest-mismatched intents and a +revocation before its final metadata write. These fixtures do not supply production +runtime authority by themselves. + +Pre-origin ownership proof, Git journal equivalence, safe reference-aware cleanup, +complete CLI/TUI/App recovery, remaining crash cuts and platform acceptance remain +required. M2–M8 and both issues stay open; the feature is not merge-ready. + + +Final local macOS arm64 verification: 407 server unit tests passed with one ignored +subprocess fixture; all 22 affected integration cases passed (4 idle-cancellation, +13 Location, 5 rewind-preview). All 73 snapshot cases passed (38 unit, 18 file-copy, +17 Git), including the native cancellation/digest/incomplete-intent regression. +All 14 real CLI branch/recovery cases passed, including the two actual partial- +restore kill scenarios and the new ordinary acknowledgement fixture. SDK typecheck +and generation consistency passed; 132 SDK tests passed and 10 configured process +tests were skipped because `CYBER_BIN` was unset. OpenAPI consistency, formatting, +diff checks and server/snapshot/CLI all-target Clippy with warnings denied passed. +Strict pinned OpenSpec passed all 117 items; lint reported 0 errors/25 warnings, +and inventory remains 53 capabilities/1030 requirements. Native sandbox deprecation +and Darwin linker warnings were observed without assigning their origin to main. +Linux/Windows, full-workspace and TUI/App recovery acceptance were not run here. + +During verification the new Store fixture initially used the wrong options type, +then installed its lost-reply hook on a reader rather than the actual writer. The +corrected writer hook proves an actual durable paired commit with a lost reply and +no notification. The new native fixture initially omitted required progress from +its completed journal; it now performs all native installs and records valid +completion before testing metadata-only acknowledgement. The SDK regression also +caught the optional response wrapper contract; missing receipt lookup now returns +404 consistently with recovery-request lookup. Final tests above supersede those +failed fixture/contract runs. + +The code-review skill was applied across the changed runtime/native/HTTP/test/spec +and generated SDK files. The review checks Session/activity key equality, native +origin/type separation, read-transaction consistency, sequence bounds and batch +identity, current authority/cancellation, exact native locks and namespace, and the +absence of source reads/replay during acknowledgement. Lookup checks both requested +owner keys against the receipt, and even an empty batch requires its original +Session sequence anchor. The repository runtime port/native adapter separation is +preserved. No critical/major findings remain for this increment; broader feature +acceptance gaps remain recorded above. A remote refresh reports main `db9e171e`, +issues #92/#93 open and no open PRs. This increment is local, not pushed or merged. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 8b1a16ed6..d11fb6a52 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -196,5 +196,19 @@ an explicit metadata-only acknowledgement operation. New durable receipts bind a immutable resolution digest; older receipts lacking it cannot authorize new journal writes. The worker observes the exact committed owner pair, retains native locks, and checks current authority before publishing child and parent acknowledgements. -Lookup remains read-only. Ordinary restore reconciliation, pre-origin cuts, Git -integration and reference-aware reclamation remain pending implementation. +Lookup remains read-only. Ordinary file-copy restore reconciliation now uses a receipt binding the immutable +intent and exact paired Session event batch. Pre-origin cuts, Git integration and +reference-aware reclamation remain pending implementation. + + +Ordinary restore acknowledgements use a distinct `SnapshotRestoreReceipt`, stored +on the exact native activity in the same checked append as the Session changes. +The receipt binds the original native origin, immutable intent digest, bounded +Session sequence range and a digest of that range's event types, payloads and +causation IDs. Reconciliation observes this pair in a single database snapshot, +then reacquires the original native namespace and lock. It may publish only the +acknowledgement flag after current authority checks; it never inspects/replaces +later owner files. Historical native settlements lacking this receipt remain +inspectable but cannot acquire new acknowledgement-write authority. A stale +in-memory Session after a committed-but-lost writer reply must be reloaded before +this operation can proceed. This does not manufacture a second transcript change. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 5bfd36706..963ffca0d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -105,6 +105,17 @@ - **WHEN** a managed journal records filesystem completion but its runtime commit or journal acknowledgement is unobserved - **THEN** the retained evidence blocks another restore and is not treated as successful recovery or eligible for deletion +#### Scenario: Ordinary restore loses its acknowledgement reply +- **WHEN** an ordinary file-copy restore's paired Session/native commit is durable but its reply or journal acknowledgement is lost +- **THEN** lookup proves its original immutable intent and exact Session event batch without redispatching restore +- **AND** explicit acknowledgement rechecks current runtime authority and original native namespace/lock under retained worker owners, changes only journal acknowledgement metadata, and preserves later owner files, symlinks and transcript state +- **AND** stale in-memory Session state requires reload, historical settlements without the persisted immutable receipt cannot authorize new writes, and unrelated unknown activities retain their own fences + +#### Scenario: Recovery request reconciles journal acknowledgement +- **WHEN** both exact recovery owner records contain the durable consuming receipt but a journal acknowledgement is missing +- **THEN** explicit reconciliation verifies the persisted immutable resolution digest and original intent, publishes child acknowledgement before the parent's settlement link, and permits idempotent retry after an interrupted metadata write without inspecting or replaying later owner edits +- **AND** an uncommitted filesystem outcome or an older receipt without the digest cannot authorize new acknowledgement writes + #### Scenario: Legacy journal compatibility - **WHEN** inspection reads an origin-less version-1 file-copy intent - **THEN** it remains readable with its original completion semantics and is never upgraded into authority to acknowledge an unknown native activity diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 910d037c6..ea12a6f03 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -51,8 +51,7 @@ captured closed boundaries; ordinary native/hook admission is fenced at the writ Actual CLI/API service-kill cases resolve a partial namespace with keep and rollback, preserve unrelated/later owner files and the transcript, and repeat only lookup after restart. This is local non-Git evidence, not complete operator/client -or platform acceptance. Typed CLI/TUI recovery, ordinary-restore acknowledgement reconciliation, -pre-origin cuts and Git-native equivalents remain required. Explicit recovery-request +or platform acceptance. Typed CLI/TUI recovery, pre-origin cuts and Git-native equivalents remain required. Explicit recovery-request acknowledgement reconciliation now uses the exact committed pair and persisted immutable resolution digest; it changes only journal metadata under retained native locks and never rescans or reinstalls source files. @@ -74,7 +73,10 @@ acknowledgement with missing journal acknowledgement from an uncommitted filesys outcome. Regression coverage refuses the latter before calling the native adapter, and preserves later owner edits, including a source replaced with a symlink. Older receipts without an immutable resolution digest cannot authorize new journal -writes. Ordinary restore receipts still require their own reconciliation path. +writes. New ordinary file-copy restores now persist a separate immutable intent/Session-batch +receipt in the paired append. Its lookup and metadata-only acknowledgement path +checks the exact batch without replay. Historical settlements lacking that proof +cannot authorize new journal writes. Unprepared reservation/origin/intent cuts still require exact resource ownership proof; missing records alone do not license acknowledgement. @@ -88,7 +90,7 @@ The non-Git reviewed-confirm path now uses a runtime-issued synchronous worker, origin-bound intent and paired Session/native commit followed by journal acknowledgement. The recovery path adds globally reserved request ownership and paired original/resolution settlement. M2–M8 remain open: Git and legacy entry-point -integration, pre-origin resource proof, ordinary-restore journal reconciliation, cleanup, +integration, pre-origin resource proof, cleanup, complete clients and native/platform acceptance are still required. New version-3 file-copy intents retain native per-change source/staged file and diff --git a/sdk/openapi.json b/sdk/openapi.json index bdef31eec..ec4da20fa 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7485,6 +7485,48 @@ "filesystem_complete" ] }, + "Data_SnapshotRestoreReceipt": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/SnapshotRestoreReceipt" + } + }, + "required": [ + "data" + ] + }, + "SnapshotRestoreReceipt": { + "type": "object", + "additionalProperties": false, + "properties": { + "origin": { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + "intent_digest": { + "type": "string" + }, + "session_before": { + "type": "integer", + "format": "int64" + }, + "session_after": { + "type": "integer", + "format": "int64" + }, + "events_digest": { + "type": "string" + } + }, + "required": [ + "origin", + "intent_digest", + "session_before", + "session_after", + "events_digest" + ], + "description": "Exact native intent and Session batch consumed by an ordinary restore.\nThis is durable acknowledgement evidence, never authority to replay files." + }, "RewindSelection": { "type": "object", "additionalProperties": false, @@ -24569,6 +24611,112 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/restores/{activityID}": { + "get": { + "operationId": "v1.session.snapshotRestoreLookup", + "summary": "Read the exact committed ordinary restore receipt without native access or replay", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "activityID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotRestoreReceipt" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, + "/api/v1/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge": { + "post": { + "operationId": "v1.session.snapshotRestoreAcknowledge", + "summary": "Acknowledge the immutable ordinary restore intent after its paired Session commit without replaying source files", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "activityID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotRestoreReceipt" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/preview": { "post": { "operationId": "v1.session.rewindPreview", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index 81e61f4b7..a1e6abbdb 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1583,6 +1583,18 @@ export interface Operations { body: S.SnapshotRecoverySelection; response: S.SnapshotRecoveryReview; }; + "v1.session.snapshotRestoreAcknowledge": { + path: { sessionID: string; activityID: string }; + query: Record; + body: undefined; + response: S.SnapshotRestoreReceipt; + }; + "v1.session.snapshotRestoreLookup": { + path: { sessionID: string; activityID: string }; + query: Record; + body: undefined; + response: S.SnapshotRestoreReceipt; + }; "v1.session.startDelegation": { path: { sessionID: string; requestID: string }; query: Record; @@ -4716,6 +4728,30 @@ export const operations = { unwrap: true, stream: false, }, + "v1.session.snapshotRestoreAcknowledge": { + tag: "session", + name: "snapshotRestoreAcknowledge", + method: "POST", + path: "/api/v1/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge", + pathParams: ["sessionID", "activityID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, + "v1.session.snapshotRestoreLookup": { + tag: "session", + name: "snapshotRestoreLookup", + method: "GET", + path: "/api/v1/sessions/{sessionID}/rewind/restores/{activityID}", + pathParams: ["sessionID", "activityID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, "v1.session.startDelegation": { tag: "session", name: "startDelegation", @@ -6121,6 +6157,12 @@ export function createGroups(caller: Caller) { /** Review retained snapshot effects without restoring, acknowledging native activity or starting inference (`POST /api/v1/sessions/{sessionID}/rewind/recovery/review`) */ snapshotRecoveryReview: (sessionID: string, body: Operations["v1.session.snapshotRecoveryReview"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecoveryReview", { path: { sessionID }, body }, options), + /** Acknowledge the immutable ordinary restore intent after its paired Session commit without replaying source files (`POST /api/v1/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge`) */ + snapshotRestoreAcknowledge: (sessionID: string, activityID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotRestoreAcknowledge", { path: { sessionID, activityID } }, options), + /** Read the exact committed ordinary restore receipt without native access or replay (`GET /api/v1/sessions/{sessionID}/rewind/restores/{activityID}`) */ + snapshotRestoreLookup: (sessionID: string, activityID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotRestoreLookup", { path: { sessionID, activityID } }, options), /** Durably reserve a named user delegation request (`POST /api/v1/sessions/{sessionID}/delegations/{requestID}`) */ startDelegation: (sessionID: string, requestID: string, body: Operations["v1.session.startDelegation"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.startDelegation", { path: { sessionID, requestID }, body }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index ef1a23bf8..b2734420d 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -1114,6 +1114,10 @@ export type Data_SnapshotRecoveryReview = { data: SnapshotRecoveryReview; }; +export type Data_SnapshotRestoreReceipt = { + data: SnapshotRestoreReceipt; +}; + export type Data_SubtreeReopenReport = { data: SubtreeReopenReport; }; @@ -4805,6 +4809,18 @@ export type SnapshotRecoverySelection = { export type SnapshotRecoveryState = "pending" | "unknown" | "acknowledged" | "resolved_by_another_request"; +/** + * Exact native intent and Session batch consumed by an ordinary restore. + * This is durable acknowledgement evidence, never authority to replay files. + */ +export type SnapshotRestoreReceipt = { + origin: SnapshotOrigin; + intent_digest: string; + session_before: number; + session_after: number; + events_digest: string; +}; + export type SolveCandidateEvidence = { candidate_id: string; label?: string | null; diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index becd7b846..a109ac7d1 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -58,3 +58,16 @@ test("snapshot recovery acknowledgement uses the retained request without a repl assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/requests/request%2Fwith%20space/acknowledge"); assert.ok(!calls[0]!.body); }); + +test("ordinary restore lookup and acknowledgement preserve immutable paired receipt identity", async () => { + const receipt = { origin: { activity_id: "op_ordinary" }, intent_digest: "d".repeat(64), + session_before: 9, session_after: 11, events_digest: "e".repeat(64) }; + const { client, calls } = mockClient(() => json(200, { data: receipt })); + assert.deepEqual(await client.session.snapshotRestoreLookup("ses_owner", "op_ordinary"), receipt); + assert.deepEqual(await client.session.snapshotRestoreAcknowledge("ses_owner", "op_ordinary"), receipt); + assert.equal(calls[0]!.method, "GET"); + assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/restores/op_ordinary"); + assert.equal(calls[1]!.method, "POST"); + assert.equal(calls[1]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/restores/op_ordinary/acknowledge"); + assert.ok(!calls[1]!.body); +}); From fd68e5cf5f7fa5355a5b81b256d49095216ed2a7 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 21:02:21 -0400 Subject: [PATCH 10/87] Bind snapshot reservations to retained native lifetime locks --- crates/cyber-cli/tests/branches_cli.rs | 18 + crates/cyber-server/src/runtime/activity.rs | 22 +- crates/cyber-server/src/runtime/host.rs | 8 + crates/cyber-server/src/runtime/mod.rs | 2 + .../src/runtime/snapshot_lifetime.rs | 86 ++++ .../src/runtime/snapshot_recovery_record.rs | 1 + .../cyber-server/src/runtime/snapshot_work.rs | 19 +- .../src/runtime/snapshot_work_tests.rs | 170 ++++++++ crates/cyber-snapshot/src/copy/lifetime.rs | 383 ++++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 26 ++ crates/cyber-snapshot/src/lib.rs | 37 ++ .../implementation.md | 73 ++++ .../managed-recovery.md | 30 +- .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 10 + 15 files changed, 886 insertions(+), 5 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_lifetime.rs create mode 100644 crates/cyber-snapshot/src/copy/lifetime.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index d665b02f6..30695d65a 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1198,6 +1198,7 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { .unwrap(); let activity: Value = serde_json::from_str(&data).unwrap(); assert_eq!(activity["snapshot_origin"], *origin); + assert_snapshot_lifetime(&namespaces[0], &activity, origin); assert_eq!(activity["status"], "settled"); assert_eq!(activity["phase"], "snapshot_restore_committed"); let created: String = db @@ -1210,6 +1211,23 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { assert_eq!(origin["session_created_id"], created); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_snapshot_lifetime(namespace: &std::path::Path, activity: &Value, origin: &Value) { + let lifetime = &activity["snapshot_lifetime"]; + assert_eq!(lifetime["version"], 1); + assert_eq!(lifetime["owner"]["activity_id"], origin["activity_id"]); + assert_eq!( + lifetime["owner"]["session_created_id"], + origin["session_created_id"] + ); + let lifetime_file = namespace + .join("workers") + .join(origin["activity_id"].as_str().unwrap()); + let recorded_lifetime: Value = + serde_json::from_slice(&std::fs::read(lifetime_file).unwrap()).unwrap(); + assert_eq!(recorded_lifetime, *lifetime); +} + #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn reviewed_non_git_cap_warns_once_without_failing_the_conversation() { diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 7f1197395..48225e25d 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -36,6 +36,8 @@ pub(super) struct Record { #[serde(default, skip_serializing_if = "Option::is_none")] pub snapshot_restore_receipt: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_lifetime: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_bindings: Option>, } pub(super) struct Scope { @@ -94,13 +96,22 @@ impl Scope { binary_document: Option, ) -> Result { let source = handle.state.lock().await.info.id.clone(); - Self::reserve_owner(inner, source, binary_document, false).await + Self::reserve_owner(inner, source, binary_document, false, None).await } + #[cfg(all(test, any(target_os = "macos", target_os = "linux")))] pub(super) async fn reserve_snapshot( inner: &Inner, source: String, ) -> Result { - Self::reserve_owner(inner, source, None, true).await + Self::reserve_owner(inner, source, None, true, None).await + } + pub(super) async fn reserve_snapshot_owned( + inner: &Inner, + source: String, + id: String, + lifetime: Option, + ) -> Result { + Self::reserve_owner(inner, source, None, true, Some((id, lifetime))).await } pub(super) fn reserve_recovery( runtime: &Runtime, @@ -108,6 +119,7 @@ impl Scope { id: String, request: super::snapshot_recovery_types::RecoveryRequest, authority: super::snapshot_recovery_authority::RecoveryAuthority, + lifetime: Option, ) -> Result { let source = request.original.session_id.clone(); let record = Record { @@ -120,6 +132,7 @@ impl Scope { snapshot_request: Some(request.clone()), snapshot_receipt: None, snapshot_restore_receipt: None, + snapshot_lifetime: lifetime, admission_bindings: Some(authority.bindings().to_vec()), }; let captured = authority.clone(); @@ -168,13 +181,14 @@ impl Scope { source: String, binary_document: Option, snapshot_worker: bool, + owner: Option<(String, Option)>, ) -> Result { let runtime = Runtime { inner: inner.me.upgrade().ok_or(RuntimeError::ShuttingDown)?, }; let authority = runtime.capture_child_admission(&source)?; let owned_handle = runtime.inner.handle(&source).await?; - let id = cyber_core::ids::new_id("op"); + let (id, lifetime) = owner.unwrap_or_else(|| (cyber_core::ids::new_id("op"), None)); let record = Record { session_id: source.clone(), status: "pending".into(), @@ -185,6 +199,7 @@ impl Scope { snapshot_request: None, snapshot_receipt: None, snapshot_restore_receipt: None, + snapshot_lifetime: lifetime, admission_bindings: Some(authority.bindings.clone()), }; inner @@ -464,6 +479,7 @@ impl Scope { || current.snapshot_worker != original.snapshot_worker || current.snapshot_origin.as_ref() != origin || current.snapshot_request != original.snapshot_request + || current.snapshot_lifetime != original.snapshot_lifetime { return Err(cyber_store::StoreError::InvalidAppend("Native activity ownership changed before disposal".into())); } diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 3af01bb77..9d9652f9c 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -1429,6 +1429,14 @@ pub struct SnapshotRecovery { /// Working-tree snapshots for a Location. pub trait Snapshots: Send + Sync { + /// Pure filesystem lifetime ownership, acquired before durable reservation. + /// Legacy hosts omit it and cannot gain pre-origin acknowledgement authority. + fn reserve_worker( + &self, + _request: super::SnapshotLifetimeRequest, + ) -> BoxFuture<'_, Result>, String>> { + Box::pin(async { Ok(None) }) + } fn recovery_review( &self, _directory: &str, diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index cd0392e62..b1dbe126c 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -127,9 +127,11 @@ mod model; mod requests; mod rewind; mod rewind_preview; +mod snapshot_lifetime; mod snapshot_recovery; mod snapshot_recovery_types; mod snapshot_restore_receipt; +pub use snapshot_lifetime::{SnapshotLifetime, SnapshotLifetimeBinding, SnapshotLifetimeRequest}; mod snapshot_work; pub use rewind_preview::{ RewindAction, RewindCodePreview, RewindConfirmation, RewindEffect, RewindPreview, diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs new file mode 100644 index 000000000..0d0cb237d --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -0,0 +1,86 @@ +//! Exact native lifetime ownership exists before a snapshot reservation is durable. +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotLifetimeRequest { + pub activity_id: String, + pub session_id: String, + pub session_created_id: String, + pub directory: PathBuf, + pub worktree_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotLifetimeBinding { + pub version: u32, + pub owner: SnapshotLifetimeRequest, + pub root_identity: String, + pub namespace_identity: String, + pub directory_identity: String, + pub file_identity: String, +} + +/// The actual worker retains this lock through all resource acquisition and work. +/// It conveys no authority to settle checkout, process or binary-document owners. +pub trait SnapshotLifetime: Send { + fn binding(&self) -> &SnapshotLifetimeBinding; + fn verify(&self) -> Result<(), String>; +} + +impl super::Runtime { + pub(super) async fn snapshot_lifetime( + &self, + state: &super::SessionState, + activity_id: &str, + ) -> Result>, super::RuntimeError> { + let created = self + .inner + .store + .read_events(&state.info.id, -1, 1)? + .events + .first() + .ok_or_else(|| { + super::RuntimeError::Conflict("Missing snapshot Session creation".into()) + })? + .id + .clone(); + let request = SnapshotLifetimeRequest { + activity_id: activity_id.to_owned(), + session_id: state.info.id.clone(), + session_created_id: created, + directory: std::fs::canonicalize(&state.info.directory) + .map_err(|error| super::RuntimeError::Invalid(error.to_string()))?, + worktree_id: state.info.worktree_id.clone(), + }; + let lifetime = self + .inner + .options + .snapshots + .reserve_worker(request.clone()) + .await + .map_err(super::RuntimeError::Conflict)?; + if let Some(lifetime) = &lifetime { + let binding = lifetime.binding(); + if binding.version != 1 + || binding.owner != request + || [ + &binding.root_identity, + &binding.namespace_identity, + &binding.directory_identity, + &binding.file_identity, + ] + .iter() + .any(|identity| identity.is_empty() || identity.len() > 256) + { + return Err(super::RuntimeError::Conflict( + "Snapshot lifetime ownership differs".into(), + )); + } + lifetime.verify().map_err(super::RuntimeError::Conflict)?; + } + Ok(lifetime) + } +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_record.rs b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs index 5cf388965..9fa837f5d 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_record.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs @@ -170,6 +170,7 @@ pub(super) fn attest( || record.snapshot_origin.as_ref() != Some(origin) }) || launched.snapshot_request != latest.snapshot_request + || launched.snapshot_lifetime != latest.snapshot_lifetime || !matches!(latest.status.as_str(), "pending" | "unknown" | "settled") { return Err(StoreError::InvalidAppend( diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index e43a55e25..399113202 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -98,6 +98,8 @@ pub struct SnapshotWorker { worktree_id: Option, _location: Box, _lifecycle: OwnedRwLockReadGuard<()>, + // Last: releasing this lock proves all earlier worker resources were dropped. + _lifetime: Option>, } struct SnapshotAdmission(Arc); @@ -135,6 +137,9 @@ impl SnapshotWork<'_> { } } pub fn authorize(&self) -> Result<(), String> { + if let Some(lifetime) = &self.owner._lifetime { + lifetime.verify()?; + } if self.owner.scope.control.stop.is_cancelled() { return Err("Snapshot worker cancelled; retain its journal before retrying".into()); } @@ -332,12 +337,14 @@ impl Runtime { &request.original.session_created_id, &request.confirmation.request_id, ); + let lifetime = self.snapshot_lifetime(state, &id).await?; let scope = activity::Scope::reserve_recovery( self, handle.clone(), id, request.clone(), authority, + lifetime.as_ref().map(|owner| owner.binding().clone()), )?; let lease = self .inner @@ -372,6 +379,7 @@ impl Runtime { worktree_id: state.info.worktree_id.clone(), _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, _lifecycle: lifecycle, + _lifetime: lifetime, }) } pub(crate) async fn snapshot_worker( @@ -389,7 +397,15 @@ impl Runtime { .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) .map_err(|_| RuntimeError::Busy(state.info.id.clone()))?; let admission = SnapshotAdmission(handle.clone()); - let scope = activity::Scope::reserve_snapshot(&self.inner, state.info.id.clone()).await?; + let id = cyber_core::ids::new_id("op"); + let lifetime = self.snapshot_lifetime(state, &id).await?; + let scope = activity::Scope::reserve_snapshot_owned( + &self.inner, + state.info.id.clone(), + id, + lifetime.as_ref().map(|owner| owner.binding().clone()), + ) + .await?; let lease = self .inner .claim_location(&state.info, false, scope.control.stop.clone()) @@ -419,6 +435,7 @@ impl Runtime { worktree_id, _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, _lifecycle: lifecycle, + _lifetime: lifetime, admission, sequence: state.last_seq, }) diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 82ebc7565..4748ff96e 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -89,6 +89,176 @@ fn runtime_with_snapshots( } struct Acknowledgements(Arc); +struct Lifetimes { + inner: Acknowledgements, + permits: Arc, + foreign: bool, +} +struct LifetimePermit { + binding: super::super::SnapshotLifetimeBinding, + _permit: tokio::sync::OwnedSemaphorePermit, +} +impl super::super::SnapshotLifetime for LifetimePermit { + fn binding(&self) -> &super::super::SnapshotLifetimeBinding { + &self.binding + } + fn verify(&self) -> Result<(), String> { + Ok(()) + } +} +impl Snapshots for Lifetimes { + fn reserve_worker( + &self, + mut request: super::super::SnapshotLifetimeRequest, + ) -> futures::future::BoxFuture< + '_, + Result>, String>, + > { + Box::pin(async move { + let permit = self + .permits + .clone() + .acquire_owned() + .await + .map_err(|error| error.to_string())?; + if self.foreign { + request.session_id = "ses_foreign".into(); + } + Ok(Some(Box::new(LifetimePermit { + binding: super::super::SnapshotLifetimeBinding { + version: 1, + owner: request, + root_identity: "fixture-root".into(), + namespace_identity: "fixture-namespace".into(), + directory_identity: "fixture-directory".into(), + file_identity: "fixture-file".into(), + }, + _permit: permit, + }) + as Box)) + }) + } + fn track( + &self, + directory: &str, + ) -> futures::future::BoxFuture<'_, Result, String>> { + self.inner.track(directory) + } + fn changed( + &self, + directory: &str, + before: &str, + after: &str, + ) -> futures::future::BoxFuture<'_, Result, String>> { + self.inner.changed(directory, before, after) + } + fn diff( + &self, + directory: &str, + before: &str, + after: &str, + ) -> futures::future::BoxFuture<'_, Result, String>> { + self.inner.diff(directory, before, after) + } + fn restore( + &self, + directory: &str, + before: &str, + after: &str, + ) -> futures::future::BoxFuture<'_, Result, RestoreError>> { + self.inner.restore(directory, before, after) + } +} + +#[tokio::test] +async fn pre_origin_lifetime_is_bound_before_reservation_and_retained_through_request_disposal() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: permits.clone(), + foreign: false, + }), + ); + let owner = worker(&runtime, &session).await; + let id = owner.scope.id().to_owned(); + assert_eq!(permits.available_permits(), 0); + let record = latest(&runtime, &session); + assert!(record["snapshot_origin"].is_null()); + assert_eq!(record["snapshot_lifetime"]["owner"]["activity_id"], id); + assert_eq!(record["snapshot_lifetime"]["owner"]["session_id"], session); + let (started, running) = tokio::sync::oneshot::channel(); + let (finish, gate) = std::sync::mpsc::channel(); + let task = tokio::spawn(owner.run(move |work| { + work.authorize().unwrap(); + started.send(()).unwrap(); + gate.recv().unwrap(); + assert!(work.effect(|| Ok(())).is_err()); + Err(RestoreError::Failed( + "cancelled pre-origin preparation".into(), + )) + })); + running.await.unwrap(); + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + assert_eq!(permits.available_permits(), 0); + let mut shutdown = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.shutdown().await } + }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut shutdown) + .await + .is_err() + ); + finish.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), shutdown) + .await + .unwrap() + .unwrap(); + assert_eq!(permits.available_permits(), 1); + base.shutdown().await; +} + +#[tokio::test] +async fn foreign_lifetime_proof_is_rejected_before_any_native_reservation() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: permits.clone(), + foreign: true, + }), + ); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + assert!(runtime.snapshot_worker(&handle, &state).await.is_err()); + drop(state); + assert_eq!(permits.available_permits(), 1); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); + let count: i64 = runtime + .inner + .store + .read(|db| { + Ok(db.query_row( + "SELECT COUNT(*) FROM event WHERE type='native.activity.changed.1'", + [], + |row| row.get(0), + )?) + }) + .unwrap(); + assert_eq!(count, 0); + runtime.shutdown().await; + base.shutdown().await; +} impl Snapshots for Acknowledgements { fn track( &self, diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs new file mode 100644 index 000000000..daff83c07 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -0,0 +1,383 @@ +//! A worker-specific native lock precedes every durable runtime reservation. +use super::{Options, Store, storage}; +use cyber_server::runtime::{SnapshotLifetime, SnapshotLifetimeBinding, SnapshotLifetimeRequest}; +use std::{ + fs::File, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +const RECORD_LIMIT: u64 = 16 * 1024; + +pub(crate) struct Lifetime { + binding: SnapshotLifetimeBinding, + file: File, + data: PathBuf, + options: Options, +} +impl Drop for Lifetime { + fn drop(&mut self) { + let _ = self.file.unlock(); + } +} + +impl SnapshotLifetime for Lifetime { + fn binding(&self) -> &SnapshotLifetimeBinding { + &self.binding + } + fn verify(&self) -> Result<(), String> { + if storage::identity(&self.file)? != self.binding.file_identity { + return Err("Snapshot lifetime descriptor changed".into()); + } + storage::validate_lock(&self.file)?; + let store = Store::existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Snapshot lifetime namespace is missing")?; + store.verify_lifetime(&self.binding) + } +} + +fn lock(file: &File) -> Result<(), String> { + file.try_lock().map_err(|error| match error { + std::fs::TryLockError::WouldBlock => "Snapshot worker is still active".into(), + error => error.to_string(), + }) +} + +fn valid_owner(owner: &SnapshotLifetimeRequest) -> Result<(), String> { + if owner.directory.to_str().is_none() + || !owner.activity_id.starts_with("op_") + || owner.activity_id.len() > 256 + || !owner + .activity_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + || [&owner.session_id, &owner.session_created_id] + .iter() + .any(|id| id.is_empty() || id.len() > 256) + { + return Err("Invalid snapshot lifetime owner".into()); + } + Ok(()) +} + +impl Store { + pub(crate) fn reserve_lifetime( + &self, + owner: SnapshotLifetimeRequest, + ) -> Result { + self.reserve_lifetime_before_publish(owner, || Ok(())) + } + fn reserve_lifetime_before_publish( + &self, + owner: SnapshotLifetimeRequest, + before_publish: impl FnOnce() -> Result<(), String>, + ) -> Result { + valid_owner(&owner)?; + if owner.directory != self.root { + return Err("Snapshot lifetime Location changed".into()); + } + let directory = storage::private_child(&self.namespace, "workers")?; + if directory.symlink_metadata(&owner.activity_id).is_ok() { + let bytes = storage::read(&directory, &owner.activity_id, RECORD_LIMIT)?; + let binding: SnapshotLifetimeBinding = + serde_json::from_slice(&bytes).map_err(storage::fail)?; + if binding.owner != owner { + return Err("Snapshot lifetime owner changed".into()); + } + return self.lock_lifetime(&binding); + } + let mut binding = SnapshotLifetimeBinding { + version: 1, + owner, + root_identity: self.root_id.clone(), + namespace_identity: storage::dir_identity(&self.namespace)?, + directory_identity: storage::dir_identity(&directory)?, + file_identity: "0".repeat(storage::IDENTITY_MAX_BYTES), + }; + let budget = serde_json::to_vec(&binding).map_err(storage::fail)?.len() as u64; + if budget > RECORD_LIMIT { + return Err("Snapshot lifetime record exceeds limit".into()); + } + self.reserve(budget)?; + // A crash during construction retains an unreferenced private stage; + // the stable owner name is published only with a complete durable record. + let stage = format!("preparing_{}", ulid::Ulid::new()); + let mut file = storage::create(&directory, &stage, b"")?; + lock(&file)?; + binding.file_identity = storage::identity(&file)?; + let bytes = serde_json::to_vec(&binding).map_err(storage::fail)?; + file.write_all(&bytes).map_err(storage::fail)?; + storage::sync_file(&file)?; + before_publish()?; + rustix::fs::renameat_with( + &directory, + &stage, + &directory, + &binding.owner.activity_id, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(storage::fail)?; + storage::sync(&directory)?; + Ok(Lifetime { + binding, + file, + data: self.data_path.clone(), + options: self.options.clone(), + }) + } + + /// Reacquire only a persisted exact lock. Never create/replace missing proof. + pub(crate) fn lock_lifetime( + &self, + binding: &SnapshotLifetimeBinding, + ) -> Result { + self.verify_lifetime(binding)?; + let directory = storage::existing_child(&self.namespace, "workers", true)?; + let file = storage::open(&directory, Path::new(&binding.owner.activity_id))?; + storage::validate_lock(&file)?; + lock(&file)?; + self.verify_lifetime(binding)?; + if storage::identity(&file)? != binding.file_identity { + return Err("Snapshot lifetime lock replaced".into()); + } + Ok(Lifetime { + binding: binding.clone(), + file, + data: self.data_path.clone(), + options: self.options.clone(), + }) + } + + fn verify_lifetime(&self, binding: &SnapshotLifetimeBinding) -> Result<(), String> { + valid_owner(&binding.owner)?; + self.verify_root()?; + if binding.version != 1 + || binding.owner.directory != self.root + || binding.root_identity != self.root_id + || binding.namespace_identity != storage::dir_identity(&self.namespace)? + { + return Err("Snapshot lifetime namespace changed".into()); + } + let directory = storage::existing_child(&self.namespace, "workers", true)?; + let file = storage::open(&directory, Path::new(&binding.owner.activity_id))?; + storage::validate_lock(&file)?; + let mut bytes = Vec::new(); + file.try_clone() + .map_err(storage::fail)? + .take(RECORD_LIMIT + 1) + .read_to_end(&mut bytes) + .map_err(storage::fail)?; + if bytes.len() as u64 > RECORD_LIMIT { + return Err("Snapshot lifetime record exceeds limit".into()); + } + let observed: SnapshotLifetimeBinding = + serde_json::from_slice(&bytes).map_err(storage::fail)?; + if binding.directory_identity != storage::dir_identity(&directory)? + || binding.file_identity != storage::identity(&file)? + || observed != *binding + { + return Err("Snapshot lifetime binding changed".into()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn options() -> Options { + Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + } + } + fn request(root: &Path) -> SnapshotLifetimeRequest { + SnapshotLifetimeRequest { + activity_id: "op_lifetime_fixture".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: root.canonicalize().unwrap(), + worktree_id: None, + } + } + fn store(root: &Path) -> Store { + std::fs::create_dir_all(root.join("work")).unwrap(); + Store::open( + &root.join("data"), + root.join("work").to_str().unwrap(), + options(), + ) + .unwrap() + } + + #[test] + fn exact_lifetime_lock_retains_privacy_quota_and_rejects_replacement_or_missing_proof() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let lifetime = store.reserve_lifetime(request(&store.root)).unwrap(); + let binding = lifetime.binding().clone(); + let path = temp.path().join("data/snapshot/global"); + let namespace = std::fs::read_dir(path) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let file = namespace.join("workers").join(&binding.owner.activity_id); + assert_eq!( + std::fs::metadata(&file).unwrap().permissions().mode() & 0o777, + 0o600 + ); + assert!(store.usage().unwrap() >= std::fs::metadata(&file).unwrap().len()); + assert!(store.lock_lifetime(&binding).is_err()); + lifetime.verify().unwrap(); + let alias = namespace.join("lifetime_alias"); + std::fs::hard_link(&file, &alias).unwrap(); + assert!(lifetime.verify().is_err()); + assert!(store.lock_lifetime(&binding).is_err()); + std::fs::remove_file(alias).unwrap(); + lifetime.verify().unwrap(); + drop(lifetime); + let recovered = store.lock_lifetime(&binding).unwrap(); + let bytes = std::fs::read(&file).unwrap(); + let replacement = namespace.join("replacement"); + std::fs::write(&replacement, bytes).unwrap(); + std::fs::set_permissions(&replacement, std::fs::Permissions::from_mode(0o600)).unwrap(); + std::fs::rename(replacement, &file).unwrap(); + assert!(recovered.verify().is_err()); + assert!(store.lock_lifetime(&binding).is_err()); + std::fs::remove_file(&file).unwrap(); + assert!(store.lock_lifetime(&binding).is_err()); + assert!(!file.exists()); + } + + #[test] + fn lifetime_cap_refuses_before_creating_an_owner_file_and_gc_retains_preparation() { + let temp = tempfile::tempdir().unwrap(); + let mut store = store(temp.path()); + store.options.max_total = 1; + assert!(store.reserve_lifetime(request(&store.root)).is_err()); + let workers = storage::existing_child(&store.namespace, "workers", true).unwrap(); + assert!(storage::names(&workers, 10).unwrap().is_empty()); + store.options.max_total = 4 * 1024 * 1024; + let lifetime = store.reserve_lifetime(request(&store.root)).unwrap(); + let object = store.object(b"protected preparation bytes").unwrap(); + let config: std::sync::Arc = + std::sync::Arc::new(|_| serde_json::json!({"snapshots":{"retention_days":0}})); + super::super::gc_location(&store.namespace, &config, &Default::default()).unwrap(); + assert!(storage::open(&store.objects, Path::new(&object.object)).is_ok()); + lifetime.verify().unwrap(); + } + + #[test] + #[ignore = "subprocess fixture invoked by lifetime proof test"] + fn lifetime_owner_child() { + let root = PathBuf::from(std::env::var_os("CYBER_SNAPSHOT_LIFETIME_CHILD_ROOT").unwrap()); + let store = store(&root); + if std::env::var_os("CYBER_SNAPSHOT_LIFETIME_CHILD_STAGE").is_some() { + let result = store.reserve_lifetime_before_publish(request(&store.root), || { + publish_ready(&root, b"stage"); + std::io::stdin().read_exact(&mut [0]).unwrap(); + Err("injected pre-publication cut".into()) + }); + assert!(result.is_err()); + return; + } + let lifetime = store.reserve_lifetime(request(&store.root)).unwrap(); + drop(store); + publish_ready(&root, &serde_json::to_vec(lifetime.binding()).unwrap()); + let mut byte = [0]; + std::io::stdin().read_exact(&mut byte).unwrap(); + lifetime.verify().unwrap(); + } + + fn publish_ready(root: &Path, bytes: &[u8]) { + std::fs::write(root.join("ready.tmp"), bytes).unwrap(); + std::fs::rename(root.join("ready.tmp"), root.join("ready.json")).unwrap(); + } + + #[test] + fn real_process_death_before_publication_retains_stage_without_poisoning_the_owner_name() { + use std::process::{Command, Stdio}; + let temp = tempfile::tempdir().unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "copy::lifetime::tests::lifetime_owner_child", + "--ignored", + "--nocapture", + ]) + .env("CYBER_SNAPSHOT_LIFETIME_CHILD_ROOT", temp.path()) + .env("CYBER_SNAPSHOT_LIFETIME_CHILD_STAGE", "1") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !temp.path().join("ready.json").exists() && std::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("lifetime fixture exited: {status}"); + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(temp.path().join("ready.json").exists()); + child.kill().unwrap(); + child.wait().unwrap(); + let store = store(temp.path()); + let directory = storage::existing_child(&store.namespace, "workers", true).unwrap(); + let stages = storage::names(&directory, 10).unwrap(); + assert_eq!(stages.len(), 1); + assert!(stages[0].starts_with("preparing_")); + let bytes = storage::read(&directory, &stages[0], RECORD_LIMIT).unwrap(); + let lifetime = store.reserve_lifetime(request(&store.root)).unwrap(); + assert_eq!( + storage::read(&directory, &stages[0], RECORD_LIMIT).unwrap(), + bytes + ); + assert_eq!(storage::names(&directory, 10).unwrap().len(), 2); + lifetime.verify().unwrap(); + } + + #[test] + fn real_process_death_releases_only_the_exact_worker_lifetime_lock() { + use std::process::{Command, Stdio}; + let temp = tempfile::tempdir().unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "copy::lifetime::tests::lifetime_owner_child", + "--ignored", + "--nocapture", + ]) + .env("CYBER_SNAPSHOT_LIFETIME_CHILD_ROOT", temp.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let ready = temp.path().join("ready.json"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !ready.exists() && std::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("lifetime fixture exited: {status}"); + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(ready.exists()); + let binding: SnapshotLifetimeBinding = + serde_json::from_slice(&std::fs::read(ready).unwrap()).unwrap(); + let store = store(temp.path()); + assert!(store.lock_lifetime(&binding).is_err()); + child.kill().unwrap(); + child.wait().unwrap(); + let proof = store.lock_lifetime(&binding).unwrap(); + assert_eq!(proof.binding(), &binding); + proof.verify().unwrap(); + } +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 702f85eb3..b0dfbec6d 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -68,6 +68,8 @@ pub(crate) struct Store { usage: std::cell::Cell>, } +mod lifetime; + pub(crate) struct WorkerGuard { pub store: Store, pub workspace: tokio::sync::OwnedMutexGuard<()>, @@ -286,6 +288,20 @@ impl Store { .ok_or("Snapshot storage overflow")?; } } + match self.namespace.symlink_metadata("workers") { + Ok(_) => { + let workers = storage::existing_child(&self.namespace, "workers", true)?; + for name in storage::names(&workers, ENTRY_LIMIT)? { + let file = storage::open(&workers, Path::new(&name))?; + storage::validate_lock(&file)?; + size = size + .checked_add(file.metadata().map_err(fail)?.len()) + .ok_or("Snapshot storage overflow")?; + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(fail(error)), + } self.usage.set(Some(size)); Ok(size) } @@ -504,6 +520,16 @@ fn gc_location( config: &std::sync::Arc, protected: &BTreeSet, ) -> Result<()> { + match dir.symlink_metadata("workers") { + Ok(_) => { + let workers = storage::existing_child(dir, "workers", true)?; + if !storage::names(&workers, ENTRY_LIMIT)?.is_empty() { + return Ok(()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(fail(error)), + } let manifests = storage::private_child(dir, "manifests")?; let journals = storage::private_child(dir, "journals")?; // Until an owner resolves/acknowledges native evidence, no journal is evicted. diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index f077be00a..83d897eca 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -237,6 +237,43 @@ impl GitSnapshots { } impl Snapshots for GitSnapshots { + fn reserve_worker( + &self, + request: cyber_server::runtime::SnapshotLifetimeRequest, + ) -> BoxFuture<'_, Result>, String>> + { + Box::pin(async move { + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = request; + Ok(None) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + let directory = request + .directory + .to_str() + .ok_or("Invalid snapshot lifetime Location")?; + // Git journaling and its lifetime proof are integrated separately. + if cyber_core::project::worktree_root(Path::new(directory)).is_some() { + return Ok(None); + } + self.recovery_location(directory)?; + let data = self.data_dir.clone(); + let options = self.recovery_options(directory); + let directory = directory.to_owned(); + tokio::task::spawn_blocking(move || { + let store = copy::Store::open(&data, &directory, options)?; + let lifetime = store.reserve_lifetime(request)?; + Ok(Some( + Box::new(lifetime) as Box + )) + }) + .await + .map_err(|error| error.to_string())? + } + }) + } fn recovery_review( &self, directory: &str, diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 417e5a774..413838d9f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -933,3 +933,76 @@ Session sequence anchor. The repository runtime port/native adapter separation i preserved. No critical/major findings remain for this increment; broader feature acceptance gaps remain recorded above. A remote refresh reports main `db9e171e`, issues #92/#93 open and no open PRs. This increment is local, not pushed or merged. + + +### Exact lifetime evidence before snapshot origin + +The pre-origin audit found that the current Session-owned checkout lease cannot +prove exit of one snapshot worker. New file-copy workers now obtain a separate +private lifetime file lock before the runtime writes reservation sequence zero and +before it acquires checkout resources. The immutable binding includes exact native +Location/namespace/worker-directory/file identities and activity/Session-creation/ +checkout ownership. The native activity retains it unchanged through origin, +receipt settlement and late disposal; native attestation rejects a changed binding. +`SnapshotWork::authorize` revalidates the native route before effects. The guard is +the last worker field released, after its resource owners. Hosts lacking this port, +Git paths and historical reservations retain no lifetime proof. + +The record is flushed privately, then published under its stable activity name by +native no-replace rename. Interrupted private stages remain retained, without +poisoning a stable request owner name or deleting evidence to retry. Reopening an +existing worker requires the exact immutable payload and native identities; missing, +replaced, unsafe or live locks refuse. Lifetime metadata is included in the same +per-Location quota as journal/object/manifest data. GC conservatively stops pruning +while lifetime records or stages exist. This retains evidence until M6 supplies +complete reference accounting and durable two-phase reclamation; it does not claim +that reclamation or full retention/cap acceptance is implemented. + +Runtime regressions retain a real semaphore-backed lifetime through requester +abort and shutdown before an origin exists, and reject a foreign lifetime before +publishing any reservation. Native tests check private permissions, quota refusal +before owner-file creation, replacement/missing-proof refusal and conservative GC. +A native child actually holds the lifetime lock after dropping its Store: a fresh +Store refuses the live lock, then reacquires the exact identity after killing and +waiting for that process. A second process cut pauses after the staged record's +flush but before no-replace publication; its death leaves that stage intact while +a fresh worker publishes a distinct complete lock. These are storage-level cuts, +not real service/operator acceptance for the complete early-recovery protocol. +The real CLI assertion now checks that a successful ordinary rewind's runtime +record contains the same immutable lifetime payload as its private file. + +No pre-origin acknowledgement path is enabled by this increment. A missing origin +or journal remains insufficient to acknowledge checkout/process uncertainty. The +operator consuming proof, explicit source-admission phases, managed-checkout +ownership equivalence, Git integration and remaining native cuts remain required. +M2–M8 and both issues remain open. + + +Final local macOS arm64 evidence: 409 server unit tests passed (one ignored +subprocess fixture invoked by its existing parent tests), all 22 affected +integrations passed (4 idle-cancellation, 13 Location, 5 rewind-preview), and all +77 snapshot cases passed (42 unit, 18 file-copy, 17 Git). Snapshot tests list one +ignored lifetime subprocess fixture, which both new process-kill parents invoke. +All 14 real CLI branch/recovery tests passed, including ordinary rewind's persisted +lifetime assertion and actual partial-restore keep/rollback kills. SDK typecheck, +generation consistency and 132 SDK tests passed; 10 configured process tests were +skipped because `CYBER_BIN` was unset. Server/snapshot/CLI all-target Clippy with +warnings denied passed. The expanded CLI receipt assertion initially exceeded +18/15 complexity; extracting its lifetime assertion passed without allowances. +Formatting, diff checks and strict OpenSpec passed (117 items); lint reports +0 errors/25 warnings and inventory stays 53 capabilities/1030 requirements. Native +sandbox deprecation and Darwin linker warnings were observed without assigning +their origin to main. Linux/Windows, full workspace, real SDK process and TUI/App +recovery acceptance were not run. + +The code-review skill was applied to every changed runtime/native/test/spec file. +Review checks lifetime-before-reservation ordering, unchanged exact binding through +native attestation and disposal, guard teardown order, immutable payload and native +route/descriptor consistency, private modes/alias rejection, quota accounting and +reference protection. Review changed construction to staged no-replace publication +so a crash cannot leave a partial stable owner record, and reads payload and native +identity from the same descriptor during verification. The runtime/native adapter +port separation is retained. No critical/major findings remain for this prerequisite; +its explicitly missing consuming recovery protocol and broader acceptance gates +are not accepted by these tests. This is local progress, not a merged delivery or +completion of either issue. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index d11fb6a52..0c3e25281 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -11,7 +11,8 @@ acceptance; the native/platform and complete recovery tests remain required. Implementation checkpoint: the non-Git review/confirm/lookup API and SDK now use durable request reservation and paired native settlement. Local real-service partial-restore kills exercise keep/rollback and lookup without replay. Explicit -journal acknowledgement reconciliation, pre-origin resource cuts, Git journaling, +journal acknowledgement reconciliation is implemented for new file-copy receipts; +pre-origin resource cuts, Git journaling, reclamation, complete clients and the platform matrix remain unfinished. Parent tasks are open; the complete recovery delivery is not merge-ready. @@ -212,3 +213,30 @@ later owner files. Historical native settlements lacking this receipt remain inspectable but cannot acquire new acknowledgement-write authority. A stale in-memory Session after a committed-but-lost writer reply must be reloaded before this operation can proceed. This does not manufacture a second transcript change. + + +### Lifetime proof before origin publication + +New file-copy workers acquire a distinct private native lifetime lock before their +runtime reservation is published or checkout acquisition begins. Its immutable +binding names the activity, Session creation, checkout, canonical/native Location, +private namespace and exact lock file/directory identities. The reservation keeps +this binding unchanged through origin publication, settlement and disposal. The +actual worker retains the lock until its earlier resource owners are dropped; +source authorization verifies the current lifetime route as well as its existing +authority. Reacquisition of the exact native lock proves only this worker's exit. +It cannot settle a Session-owned checkout lease or another native/process owner. + +Construction writes and flushes a privately staged record before publishing its +stable owner name with no-replace. A cut before publication retains the unreferenced +stage; a later reservation can create its own complete lock without deleting that +evidence. Metadata is counted in the per-Location cap. GC retains all preparation +references conservatively while worker records exist; reference-aware reclamation +must eventually account for both complete records and orphan stages. + +This supplies lifetime evidence for the reserved-before-origin cut. It does not +by itself admit cancellation/acknowledgement of that reservation, prove that a +missing journal had no source effects, or acknowledge managed checkout/process +uncertainty. The explicit pre-effect consuming protocol, origin-before-intent cut, +Git equivalent and complete native/client acceptance remain required. Older +reservations without this binding receive no retrospective proof. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 963ffca0d..419f32032 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -93,6 +93,12 @@ - **THEN** its actual native owners remain held until the synchronous worker exits, Stop and read-only Session inspection remain available between source effects, and shutdown waits for actual ownership settlement - **AND** uncertain effects retain their origin and admission fence across runtime reload without replay +#### Scenario: Lifetime proof precedes reservation +- **WHEN** a new file-copy worker reserves runtime ownership before publishing its restore origin +- **THEN** an immutable worker-specific native lifetime lock already binds its activity, Session creation, checkout, Location and private namespace, and remains retained through actual resource teardown +- **AND** replacement, missing proof or a live lock refuses reacquisition; a privately staged incomplete publication remains retained and cannot grant acknowledgement authority +- **AND** this evidence counts toward the Location cap, protects pending references from GC, and cannot settle a generic process, Session-owned checkout lease or historical reservation without that binding + #### Scenario: Session changes before a source effect - **WHEN** the Session revision changes after preparation but before the next source mutation - **THEN** that mutation is refused and existing owner bytes remain unchanged diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index ea12a6f03..9449c182a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -84,6 +84,16 @@ proof; missing records alone do not license acknowledgement. - [ ] M7 Add owner recovery review/confirm/lookup HTTP routes, typed CLI and SDK operations, and TUI recovery entry from rewind unavailability. Regenerate OpenAPI/SDK/command artifacts; exercise the real clients with no inference or transcript movement and with stale/foreign request refusal. - [ ] M8 Run process-kill cuts before/after intent, stage preparation, every exchange, progress, filesystem completion, writer commit, journal acknowledgement and reclaim. Obtain Linux x64/ARM and macOS Intel/ARM positives/denials, explicit Windows results/limits and shared Git/binary regressions; update evidence and perform substantive code/security review before accepting parent tasks. +Lifetime continuation: new file-copy runtime reservations now retain an exact +worker-specific private lock acquired before reservation and checkout admission. +The actual worker keeps it through resource teardown, and source authorization +revalidates its native route. Lock metadata and orphan stages count toward the cap +and conservatively protect GC. Native tests reacquire only after actual process +death and preserve an interrupted pre-publication stage. This is a prerequisite, +not completion of pre-origin recovery: its consuming operator path, positive +source-admission phases, independent managed-checkout proof and Git equivalent +remain open. No new acknowledgement admission or force-clear is enabled. + Implementation source map: The non-Git reviewed-confirm path now uses a runtime-issued synchronous worker, From 8262c0f40542d8eea4912d933317896d4408b73b Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 21:28:30 -0400 Subject: [PATCH 11/87] Resolve exited snapshot preparations before origin publication --- crates/cyber-cli/tests/branches_cli.rs | 118 ++++ crates/cyber-server/src/http/openapi.rs | 8 + crates/cyber-server/src/http/sessions.rs | 64 +++ crates/cyber-server/src/runtime/activity.rs | 4 + crates/cyber-server/src/runtime/events.rs | 7 + crates/cyber-server/src/runtime/host.rs | 7 + crates/cyber-server/src/runtime/mod.rs | 5 + .../src/runtime/snapshot_lifetime.rs | 2 + .../src/runtime/snapshot_preparation.rs | 528 ++++++++++++++++++ .../src/runtime/snapshot_recovery.rs | 21 +- .../src/runtime/snapshot_recovery_record.rs | 1 + .../src/runtime/snapshot_recovery_types.rs | 1 + .../src/runtime/snapshot_work_tests.rs | 349 ++++++++++++ crates/cyber-snapshot/src/lib.rs | 31 + docs/rewind.md | 52 +- .../implementation.md | 88 +++ .../managed-recovery.md | 24 +- .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 17 +- sdk/openapi.json | 427 +++++++++++++- sdk/typescript/src/generated/operations.ts | 84 +++ sdk/typescript/src/generated/types.ts | 60 +- sdk/typescript/test/rewind.test.ts | 18 + 23 files changed, 1909 insertions(+), 13 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_preparation.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 30695d65a..fbab42c03 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -792,6 +792,124 @@ fn managed_keep_and_rollback_after_real_partial_restore_kill_preserve_owner_file managed_native_recovery_after_partial_kill("rollback"); } +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_restart() { + let model = scripted_model::ScriptedModel::start(reply); + let env = Env::new(&model); + let repo = env.root.join("repo"); + std::fs::remove_dir_all(repo.join(".git")).unwrap(); + let before = format!("b\0{}", "b".repeat(64 * 1024)); + let after = format!("a\0{}", "a".repeat(64 * 1024)); + for index in 0..1280 { + std::fs::write(repo.join(format!("bulk_{index:04}")), &before).unwrap(); + } + let id = env.exec(None, "before snapshot"); + let first = env.user_ids(&id)[0].clone(); + for index in 0..1280 { + std::fs::write(repo.join(format!("bulk_{index:04}")), &after).unwrap(); + } + env.exec(Some(&id), "after snapshot"); + let messages = env.messages(&id); + let activity = kill_before_origin(&env, &id, &first); + let requests = model.requests().len(); + let target = env.root.join("preparation-owner-target"); + std::fs::write(&target, "later owner bytes").unwrap(); + std::fs::remove_file(repo.join("bulk_0000")).unwrap(); + std::os::unix::fs::symlink(&target, repo.join("bulk_0000")).unwrap(); + let list = + json(&env.cyber(&["api", "GET", &format!("/sessions/{id}/rewind/preparations")]))["data"] + .clone(); + assert_eq!(list[0]["activity_id"], activity); + assert_eq!(list[0]["lifetime_available"], true); + let route = format!("/sessions/{id}/rewind/preparations/{activity}"); + let review = + json(&env.cyber(&["api", "POST", &format!("{route}/review"), "-d", "{}"]))["data"].clone(); + let body = + json!({"confirmation":review["confirmation"],"request_id":"before-origin"}).to_string(); + let receipt = + json(&env.cyber(&["api", "POST", &format!("{route}/confirm"), "-d", &body]))["data"] + .clone(); + kill_registered_service(&env); + let repeated = + json(&env.cyber(&["api", "POST", &format!("{route}/confirm"), "-d", &body]))["data"] + .clone(); + assert_eq!(repeated, receipt); + let lookup = + json(&env.cyber(&["api", "GET", &format!("{route}/requests/before-origin")]))["data"] + .clone(); + assert_eq!(lookup, receipt); + assert!( + std::fs::symlink_metadata(repo.join("bulk_0000")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_to_string(target).unwrap(), + "later owner bytes" + ); + assert_eq!(env.file("bulk_1279"), Some(after)); + assert_eq!(env.messages(&id), messages); + assert_eq!(model.requests().len(), requests); + env.exec(Some(&id), "after preparation cancellation"); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn kill_before_origin(env: &Env, id: &str, first: &str) -> String { + use rusqlite::OptionalExtension; + let selection = reviewed_selection(env, first, "both", "commit"); + let preview = reviewed_preview(env, id, &selection); + let mut body = selection; + body["confirmation"] = preview["confirmation"].clone(); + let path = format!("/sessions/{id}/rewind/confirm"); + let mut pending = env + .command(&["api", "POST", &path, "-d", &body.to_string()]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let database = std::fs::read_dir(env.root.join("cyber-home/data")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|extension| extension == "db")) + .unwrap(); + let db = + rusqlite::Connection::open_with_flags(database, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let activity = loop { + let found: Option = db.query_row("SELECT aggregate_id FROM event e WHERE type='native.activity.changed.1' AND seq=0 AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.snapshot_origin') IS NULL AND json_extract(data,'$.snapshot_lifetime') IS NOT NULL AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq)", [id], |row| row.get(0)).optional().unwrap(); + if let Some(found) = found { + break found; + } + assert!( + pending.try_wait().unwrap().is_none(), + "restore exited before its reservation was observed" + ); + assert!( + std::time::Instant::now() < deadline, + "pre-origin reservation was never observed" + ); + std::thread::sleep(Duration::from_millis(1)); + }; + kill_registered_service(env); + assert!(!pending.wait_with_output().unwrap().status.success()); + let data: String = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&activity], + |row| row.get(0), + ) + .unwrap(); + let record: Value = serde_json::from_str(&data).unwrap(); + assert!( + record["snapshot_origin"].is_null(), + "kill missed the pre-origin cut" + ); + activity +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn managed_native_recovery_after_partial_kill(action: &str) { let model = scripted_model::ScriptedModel::start(reply); diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index a566cdb61..845a30a5d 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -772,6 +772,14 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { .ok::>(g), op("get", "/sessions/{sessionID}/rewind/restores/{activityID}", "v1.session.snapshotRestoreLookup", "Read the exact committed ordinary restore receipt without native access or replay") .ok::>(g), + op("get", "/sessions/{sessionID}/rewind/preparations", "v1.session.snapshotPreparations", "List retained snapshot reservations without a published origin or native access") + .ok::>>(g), + op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/review", "v1.session.snapshotPreparationReview", "Review cancellation of an exited preparation using its exact native lifetime proof") + .ok::>(g), + op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/confirm", "v1.session.snapshotPreparationConfirm", "Cancel only a fingerprint-confirmed unstarted snapshot preparation without moving files or transcript") + .body::(g).ok::>(g), + op("get", "/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}", "v1.session.snapshotPreparationLookup", "Look up the original durable preparation cancellation pair without native access or replay") + .ok::>(g), op("post", "/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge", "v1.session.snapshotRestoreAcknowledge", "Acknowledge the immutable ordinary restore intent after its paired Session commit without replaying source files") .ok::>(g), op("post", "/sessions/{sessionID}/rewind/preview", "v1.session.rewindPreview", "Review a rewind without changing files or the active conversation") diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index beb737459..656abe13c 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -370,6 +370,22 @@ pub fn routes() -> Router { &format!("{s}/rewind/restores/{{activity_id}}/acknowledge"), post(snapshot_restore_acknowledge), ) + .route( + &format!("{s}/rewind/preparations"), + get(snapshot_preparations), + ) + .route( + &format!("{s}/rewind/preparations/{{activity_id}}/review"), + post(snapshot_preparation_review), + ) + .route( + &format!("{s}/rewind/preparations/{{activity_id}}/confirm"), + post(snapshot_preparation_confirm), + ) + .route( + &format!("{s}/rewind/preparations/{{activity_id}}/requests/{{request_id}}"), + get(snapshot_preparation_lookup), + ) .route(&format!("{s}/rewind/confirm"), post(rewind_confirm)) .route(&format!("{s}/revert/stage"), post(stage)) .route(&format!("{s}/revert/clear"), post(clear)) @@ -1301,6 +1317,54 @@ async fn snapshot_restore_acknowledge( .await?, })) } +async fn snapshot_preparations( + State(state): State, + Path(id): Path, +) -> Result>>> { + Ok(Json(Data { + data: state.runtime.snapshot_preparations(&id).await?, + })) +} +async fn snapshot_preparation_review( + State(state): State, + Path((id, activity)): Path<(String, String)>, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_preparation_review(&id, &activity) + .await?, + })) +} +async fn snapshot_preparation_confirm( + State(state): State, + Path((id, activity)): Path<(String, String)>, + Json(confirmation): Json, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_preparation_confirm(&id, &activity, &confirmation) + .await?, + })) +} +async fn snapshot_preparation_lookup( + State(state): State, + Path((id, activity, request)): Path<(String, String, String)>, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_preparation_lookup(&id, &activity, &request) + .await? + .ok_or_else(|| { + ApiError::not_found( + "PreparationRequestNotFoundError", + "No retained preparation cancellation request", + ) + })?, + })) +} async fn rewind_preview( State(state): State, Path(id): Path, diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 48225e25d..81bea18ea 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -38,6 +38,8 @@ pub(super) struct Record { #[serde(default, skip_serializing_if = "Option::is_none")] pub snapshot_lifetime: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_preparation_receipt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_bindings: Option>, } pub(super) struct Scope { @@ -133,6 +135,7 @@ impl Scope { snapshot_receipt: None, snapshot_restore_receipt: None, snapshot_lifetime: lifetime, + snapshot_preparation_receipt: None, admission_bindings: Some(authority.bindings().to_vec()), }; let captured = authority.clone(); @@ -200,6 +203,7 @@ impl Scope { snapshot_receipt: None, snapshot_restore_receipt: None, snapshot_lifetime: lifetime, + snapshot_preparation_receipt: None, admission_bindings: Some(authority.bindings.clone()), }; inner diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 3f06c74f7..797a83cf6 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -68,6 +68,7 @@ const ALL: &[&str] = &[ CHILD_INPUT_PAUSED, super::admission_authority::FENCED, super::activity::CHANGED, + super::snapshot_preparation::CANCELLED, super::child_ownership::CHANGED, super::subtree_stop::SETTLED, super::subtree_stop::STARTED, @@ -538,6 +539,12 @@ pub fn registry() -> EventRegistry { registry .register(super::auto_statistics::RESET) .expect("valid auto statistics reset event"); + registry + .register_with( + super::snapshot_preparation::CANCELLED, + super::snapshot_preparation::validate_event, + ) + .expect("valid snapshot preparation cancellation receipt"); registry .register_with(MESSAGING_UPDATED, |data| { serde_json::from_value::(data.clone()) diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 9d9652f9c..98e7673b6 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -1429,6 +1429,13 @@ pub struct SnapshotRecovery { /// Working-tree snapshots for a Location. pub trait Snapshots: Send + Sync { + /// Read-only acquisition of an existing exact worker lifetime lock. + fn lock_worker( + &self, + _binding: super::SnapshotLifetimeBinding, + ) -> BoxFuture<'_, Result, String>> { + Box::pin(async { Err("Snapshot lifetime recovery is unavailable".into()) }) + } /// Pure filesystem lifetime ownership, acquired before durable reservation. /// Legacy hosts omit it and cannot gain pre-origin acknowledgement authority. fn reserve_worker( diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index b1dbe126c..8e8513e68 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -128,6 +128,11 @@ mod requests; mod rewind; mod rewind_preview; mod snapshot_lifetime; +mod snapshot_preparation; +pub use snapshot_preparation::{ + SnapshotPreparation, SnapshotPreparationConfirmation, SnapshotPreparationReceipt, + SnapshotPreparationReview, +}; mod snapshot_recovery; mod snapshot_recovery_types; mod snapshot_restore_receipt; diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs index 0d0cb237d..2482a4aeb 100644 --- a/crates/cyber-server/src/runtime/snapshot_lifetime.rs +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -27,6 +27,8 @@ pub struct SnapshotLifetimeBinding { /// It conveys no authority to settle checkout, process or binary-document owners. pub trait SnapshotLifetime: Send { fn binding(&self) -> &SnapshotLifetimeBinding; + /// Synchronous native metadata verification; may run inside the Store writer. + /// It must not call the runtime/Store, launch processes or perform source effects. fn verify(&self) -> Result<(), String>; } diff --git a/crates/cyber-server/src/runtime/snapshot_preparation.rs b/crates/cyber-server/src/runtime/snapshot_preparation.rs new file mode 100644 index 000000000..3af319648 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_preparation.rs @@ -0,0 +1,528 @@ +//! Consuming proof for an exited snapshot worker that never published an origin. +use super::snapshot_recovery_authority::RecoveryAuthority; +use super::{Runtime, RuntimeError, SnapshotLifetime, SnapshotLifetimeBinding, activity}; +use cyber_store::{NewEvent, StoreError}; +use rusqlite::{Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use std::sync::{Arc, PoisonError, atomic::Ordering}; + +pub(super) const CANCELLED: &str = "snapshot.preparation.cancelled.1"; +const PHASE: &str = "snapshot_preparation_cancelled"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotPreparationConfirmation { + pub confirmation: String, + pub request_id: String, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SnapshotPreparation { + pub activity_id: String, + pub status: String, + pub lifetime_available: bool, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SnapshotPreparationReview { + pub activity_id: String, + pub lifetime: SnapshotLifetimeBinding, + pub native_sequence: i64, + pub session_sequence: i64, + pub confirmation: String, + pub warning: String, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotPreparationReceipt { + pub activity_id: String, + pub request_id: String, + pub confirmation: String, + pub lifetime: SnapshotLifetimeBinding, + pub native_sequence: i64, + pub session_sequence: i64, +} + +struct Target { + sequence: i64, + raw: String, + record: activity::Record, + binding: SnapshotLifetimeBinding, +} +fn invalid(message: &str) -> StoreError { + StoreError::InvalidAppend(message.into()) +} +fn preparation_error(error: StoreError) -> RuntimeError { + match error { + StoreError::InvalidAppend(message) => RuntimeError::Conflict(message), + StoreError::Sqlite(rusqlite::Error::QueryReturnedNoRows) => { + RuntimeError::Conflict("No retained unstarted snapshot preparation".into()) + } + error => RuntimeError::Store(error), + } +} +pub(super) fn validate_event(data: &serde_json::Value) -> Result<(), String> { + let receipt: SnapshotPreparationReceipt = + serde_json::from_value(data.clone()).map_err(|error| error.to_string())?; + validate_receipt(&receipt).map_err(|error| error.to_string()) +} +fn validate_receipt(receipt: &SnapshotPreparationReceipt) -> Result<(), StoreError> { + if !(0..=1).contains(&receipt.native_sequence) + || receipt.session_sequence < 0 + || receipt.lifetime.version != 1 + || receipt.lifetime.owner.activity_id != receipt.activity_id + || receipt.request_id.is_empty() + || receipt.request_id.len() > 256 + || receipt.request_id.contains('\0') + || receipt.confirmation.len() != 64 + || !receipt + .confirmation + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(invalid("Invalid preparation cancellation receipt")); + } + Ok(()) +} +fn target(db: &Connection, session: &str, activity: &str) -> Result { + let (sequence, kind, raw): (i64, String, String) = db.query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [activity], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + let first: String = db.query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=0 AND type=?2", + [activity, activity::CHANGED], + |row| row.get(0), + )?; + let record: activity::Record = + serde_json::from_str(&raw).map_err(|_| invalid("Invalid snapshot preparation"))?; + let reserved: activity::Record = + serde_json::from_str(&first).map_err(|_| invalid("Invalid snapshot reservation"))?; + validate_unstarted(&reserved)?; + validate_unstarted(&record)?; + let binding = record + .snapshot_lifetime + .clone() + .ok_or_else(|| invalid("Historical reservation has no exact lifetime proof"))?; + if kind != activity::CHANGED + || !(0..=1).contains(&sequence) + || reserved.status != "pending" + || record.session_id != session + || reserved.session_id != session + || (sequence == 0 && record.status != "pending") + || (sequence == 1 && record.status != "unknown") + || reserved.snapshot_lifetime != record.snapshot_lifetime + || reserved.snapshot_request != record.snapshot_request + || binding.version != 1 + || binding.owner.activity_id != activity + || binding.owner.session_id != session + { + return Err(invalid( + "Snapshot reservation differs from its unstarted owner", + )); + } + let created: String = db.query_row( + "SELECT id FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq LIMIT 1", + [session, super::events::CREATED], + |row| row.get(0), + )?; + if created != binding.owner.session_created_id { + return Err(invalid("Snapshot preparation Session creation changed")); + } + Ok(Target { + sequence, + raw, + record, + binding, + }) +} +fn validate_unstarted(record: &activity::Record) -> Result<(), StoreError> { + if !record.snapshot_worker + || record.binary_document.is_some() + || record.phase != "reserved" + || record.snapshot_origin.is_some() + || record.snapshot_receipt.is_some() + || record.snapshot_restore_receipt.is_some() + || record.snapshot_preparation_receipt.is_some() + { + return Err(invalid( + "Only an exact unstarted snapshot preparation can be cancelled", + )); + } + Ok(()) +} +fn request_owner(created: &str, request: &str) -> String { + use sha2::{Digest, Sha256}; + let bytes = serde_json::to_vec(&("snapshot-preparation-request-v1", created, request)) + .expect("strings serialize"); + format!("op_snapshot_preparation_{:x}", Sha256::digest(bytes)) +} + +struct Admission(Arc); +impl Drop for Admission { + fn drop(&mut self) { + self.0.snapshot_active.store(false, Ordering::SeqCst); + } +} +struct Completion { + runtime: Runtime, + id: String, + control: Arc, +} +impl Drop for Completion { + fn drop(&mut self) { + self.runtime + .inner + .activities + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&self.id); + self.control.done.cancel(); + } +} +struct Worker { + runtime: Runtime, + handle: Arc, + authority: RecoveryAuthority, + sequence: i64, + target: Target, + control: Arc, + proof: Box, + _lifecycle: tokio::sync::OwnedRwLockReadGuard<()>, + _admission: Admission, + _completion: Completion, +} +impl Worker { + fn fingerprint(&self) -> Result { + use sha2::{Digest, Sha256}; + let bytes = serde_json::to_vec(&( + "snapshot-preparation-review-v1", + self.sequence, + &self.authority, + self.target.sequence, + &self.target.raw, + )) + .map_err(|error| RuntimeError::Invalid(error.to_string()))?; + Ok(format!("{:x}", Sha256::digest(bytes))) + } + fn authorize(&self, db: &Connection) -> Result<(), StoreError> { + if self.control.stop.is_cancelled() { + return Err(invalid("Snapshot preparation cancellation was revoked")); + } + self.proof.verify().map_err(|error| invalid(&error))?; + let owner = &self.target.binding.owner; + let current = target(db, &owner.session_id, &owner.activity_id)?; + let sequence: i64 = db.query_row( + "SELECT MAX(seq) FROM event WHERE aggregate_id=?1", + [&owner.session_id], + |row| row.get(0), + )?; + if current.raw != self.target.raw + || current.sequence != self.target.sequence + || sequence != self.sequence + || !self.authority.current(db)? + { + return Err(invalid("Snapshot preparation authority or owner changed")); + } + Ok(()) + } + async fn confirm( + self, + confirmation: SnapshotPreparationConfirmation, + ) -> Result { + if self.fingerprint()? != confirmation.confirmation { + return Err(RuntimeError::Conflict( + "Snapshot preparation review changed".into(), + )); + } + let cancelled = self.control.stop.clone().drop_guard(); + let joined = tokio::task::spawn_blocking(move || { + let receipt = SnapshotPreparationReceipt { + activity_id: self.target.binding.owner.activity_id.clone(), + request_id: confirmation.request_id, + confirmation: confirmation.confirmation, + lifetime: self.target.binding.clone(), + native_sequence: self.target.sequence, + session_sequence: self.sequence, + }; + let mut record = self.target.record.clone(); + record.status = "settled".into(); + record.phase = PHASE.into(); + record.admission_bindings = None; + record.snapshot_preparation_receipt = Some(receipt.clone()); + let events = [ + vec![NewEvent::new( + activity::CHANGED, + serde_json::to_value(record).expect("record serializes"), + )], + vec![NewEvent::new( + CANCELLED, + serde_json::to_value(&receipt).expect("receipt serializes"), + )], + ]; + let owners = [ + (receipt.activity_id.clone(), receipt.native_sequence), + ( + request_owner( + &receipt.lifetime.owner.session_created_id, + &receipt.request_id, + ), + -1, + ), + ]; + let store = self.runtime.inner.store.clone(); + // Returning the worker keeps the real proof/resources through COMMIT + // and its observed reply, rather than dropping them after the decision. + let (_, (worker, receipt)) = store + .append_pair_checked(owners, move |tx| { + self.authorize(tx)?; + Ok((events, (self, receipt))) + }) + .map_err(preparation_error)?; + worker + .handle + .acknowledge_snapshot_uncertainty(&receipt.activity_id); + Ok::<_, RuntimeError>(receipt) + }); + let result = joined.await.map_err(|_| { + RuntimeError::Conflict( + "Preparation cancellation interrupted; look up its request".into(), + ) + })?; + cancelled.disarm(); + result + } +} + +impl Runtime { + pub async fn snapshot_preparations( + &self, + session: &str, + ) -> Result, RuntimeError> { + self.session_info(session).await?; + let session = session.to_owned(); + self.inner.store.read(move |db| { + let mut query = db.prepare("SELECT aggregate_id,data FROM event e WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.status')<>'settled' AND json_extract(data,'$.snapshot_origin') IS NULL AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) ORDER BY aggregate_id LIMIT 4097")?; + let rows = query.query_map([session], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))?; + let mut result = Vec::new(); + for row in rows { + let (activity_id, data) = row?; + let record: activity::Record = serde_json::from_str(&data).map_err(|_| invalid("Invalid snapshot preparation"))?; + result.push(SnapshotPreparation { activity_id, status: record.status, lifetime_available: record.snapshot_lifetime.is_some() }); + } + if result.len() > 4096 { return Err(invalid("Snapshot preparation limit exceeded")); } + Ok(result) + }).map_err(RuntimeError::from) + } + async fn preparation_worker( + &self, + session: &str, + activity: &str, + ) -> Result { + let lifecycle = self.memory_mutation_lease().await?; + let handle = self.inner.handle(session).await?; + let state = handle.state.lock().await; + self.inner.ensure_idle(session)?; + self.ensure_no_background(session)?; + state.ensure_worktree_ready()?; + if self + .inner + .activities + .lock() + .unwrap_or_else(PoisonError::into_inner) + .values() + .any(|control| control.source == session) + { + return Err(RuntimeError::Busy(session.into())); + } + let source = session.to_owned(); + let owner = activity.to_owned(); + let target = self + .inner + .store + .read(move |db| { + let tx = db.unchecked_transaction()?; + let result = target(&tx, &source, &owner)?; + tx.commit()?; + Ok(result) + }) + .map_err(preparation_error)?; + let directory = std::fs::canonicalize(&state.info.directory) + .map_err(|error| RuntimeError::Invalid(error.to_string()))?; + if target.binding.owner.directory != directory + || target.binding.owner.worktree_id != state.info.worktree_id + { + return Err(RuntimeError::Conflict( + "Snapshot preparation Location changed".into(), + )); + } + let authority = RecoveryAuthority::capture(self, session)?; + let sequence = state.last_seq; + handle + .snapshot_active + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .map_err(|_| RuntimeError::Busy(session.into()))?; + let admission = Admission(handle.clone()); + let control = Arc::new(activity::Control { + source: session.into(), + binary_document: None, + snapshot_worker: true, + admission_sessions: super::admission_authority::binding_sessions(authority.bindings()), + stop: self.inner.closed.child_token(), + done: tokio_util::sync::CancellationToken::new(), + }); + let id = cyber_core::ids::new_id("op_preparation"); + self.inner + .activities + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(id.clone(), control.clone()); + let completion = Completion { + runtime: self.clone(), + id, + control: control.clone(), + }; + drop(state); + let proof = self + .inner + .options + .snapshots + .lock_worker(target.binding.clone()) + .await + .map_err(RuntimeError::Conflict)?; + if proof.binding() != &target.binding { + return Err(RuntimeError::Conflict( + "Foreign preparation lifetime proof".into(), + )); + } + Ok(Worker { + runtime: self.clone(), + handle, + authority, + sequence, + target, + control, + proof, + _lifecycle: lifecycle, + _admission: admission, + _completion: completion, + }) + } + pub async fn snapshot_preparation_review( + &self, + session: &str, + activity: &str, + ) -> Result { + let worker = self.preparation_worker(session, activity).await?; + let store = self.inner.store.clone(); + tokio::task::spawn_blocking(move || { + let worker = store.read(move |db| { let tx = db.unchecked_transaction()?; worker.authorize(&tx)?; tx.commit()?; Ok(worker) }).map_err(preparation_error)?; + Ok(SnapshotPreparationReview { activity_id: worker.target.binding.owner.activity_id.clone(), lifetime: worker.target.binding.clone(), + native_sequence: worker.target.sequence, session_sequence: worker.sequence, confirmation: worker.fingerprint()?, + warning: "Cancels only this unstarted snapshot preparation. Files and transcript are preserved; checkout and process uncertainty remain independent.".into() }) + }).await.map_err(|_| RuntimeError::Conflict("Snapshot preparation review interrupted".into()))? + } + pub async fn snapshot_preparation_confirm( + &self, + session: &str, + activity: &str, + confirmation: &SnapshotPreparationConfirmation, + ) -> Result { + super::snapshot_recovery::validate_request_id(&confirmation.request_id)?; + if confirmation.confirmation.len() != 64 + || !confirmation + .confirmation + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(RuntimeError::Invalid( + "Invalid preparation fingerprint".into(), + )); + } + if let Some(receipt) = self + .snapshot_preparation_lookup(session, activity, &confirmation.request_id) + .await? + { + if receipt.confirmation != confirmation.confirmation { + return Err(RuntimeError::Conflict( + "Preparation request content changed".into(), + )); + } + return Ok(receipt); + } + self.preparation_worker(session, activity) + .await? + .confirm(confirmation.clone()) + .await + } + pub async fn snapshot_preparation_lookup( + &self, + session: &str, + activity: &str, + request: &str, + ) -> Result, RuntimeError> { + super::snapshot_recovery::validate_request_id(request)?; + self.session_info(session).await?; + let source = session.to_owned(); + let activity = activity.to_owned(); + let request = request.to_owned(); + let receipt = self.inner.store.read(move |db| { + let tx = db.unchecked_transaction()?; + let created: String = tx.query_row( + "SELECT id FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq LIMIT 1", + [&source, super::events::CREATED], + |row| row.get(0), + )?; + let data: Option<(i64, String, String)> = tx + .query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [request_owner(&created, &request)], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + let Some((sequence, kind, data)) = data else { + return Ok(None); + }; + let receipt: SnapshotPreparationReceipt = + serde_json::from_str(&data).map_err(|_| invalid("Invalid preparation receipt"))?; + validate_receipt(&receipt)?; + if sequence != 0 || kind != CANCELLED + || receipt.request_id != request + || receipt.activity_id != activity + || receipt.lifetime.owner.session_id != source + || receipt.lifetime.owner.session_created_id != created + { + return Err(invalid("Preparation request belongs to another owner")); + } + let (seq, kind, data): (i64, String, String) = tx.query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&activity], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + let native: activity::Record = + serde_json::from_str(&data).map_err(|_| invalid("Invalid preparation owner"))?; + if seq != receipt.native_sequence + 1 + || kind != activity::CHANGED + || !native.snapshot_worker + || native.binary_document.is_some() + || native.session_id != source + || receipt.lifetime.owner.activity_id != activity + || native.status != "settled" + || native.phase != PHASE + || native.snapshot_preparation_receipt.as_ref() != Some(&receipt) + || native.snapshot_origin.is_some() + || native.snapshot_lifetime.as_ref() != Some(&receipt.lifetime) + { + return Err(invalid("Preparation receipt lacks its exact native pair")); + } + tx.commit()?; + Ok(Some(receipt)) + }).map_err(preparation_error)?; + if let Some(receipt) = &receipt { + self.inner + .handle(session) + .await? + .acknowledge_snapshot_uncertainty(&receipt.activity_id); + } + Ok(receipt) + } +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index 31625a712..c12ba7745 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -174,6 +174,25 @@ impl Runtime { } let state = match (record.status.as_str(), &record.snapshot_receipt) { ("pending", None) => super::SnapshotRecoveryState::Pending, + ("settled", None) if record.phase == "snapshot_preparation_cancelled" => { + let cancelled = record + .snapshot_preparation_receipt + .as_ref() + .ok_or_else(|| { + RuntimeError::Corrupt("Missing preparation cancellation receipt".into()) + })?; + if self + .snapshot_preparation_lookup(session, &id, &cancelled.request_id) + .await? + .as_ref() + != Some(cancelled) + { + return Err(RuntimeError::Corrupt( + "Preparation cancellation lacks its durable pair".into(), + )); + } + super::SnapshotRecoveryState::CancelledBeforeOrigin + } ("unknown", None) | ("settled", None) => super::SnapshotRecoveryState::Unknown, ("settled", Some(receipt)) if receipt.request_id == request_id => { super::SnapshotRecoveryState::Acknowledged @@ -317,7 +336,7 @@ impl Runtime { } } -fn validate_request_id(id: &str) -> Result<(), RuntimeError> { +pub(super) fn validate_request_id(id: &str) -> Result<(), RuntimeError> { if id.is_empty() || id.len() > 256 || id.contains('\0') { return Err(RuntimeError::Invalid( "Recovery request ID must contain 1–256 bytes without NUL".into(), diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_record.rs b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs index 9fa837f5d..03379c834 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_record.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs @@ -70,6 +70,7 @@ pub(super) fn project_pending( if !record.snapshot_worker || record.binary_document.is_some() || record.snapshot_receipt.is_some() + || record.snapshot_preparation_receipt.is_some() || original.status == "settled" || request .original diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs index 4ad5afddd..b479a21e5 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs @@ -96,6 +96,7 @@ pub enum SnapshotRecoveryState { Unknown, Acknowledged, ResolvedByAnotherRequest, + CancelledBeforeOrigin, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 4748ff96e..3bd2468bf 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -107,6 +107,23 @@ impl super::super::SnapshotLifetime for LifetimePermit { } } impl Snapshots for Lifetimes { + fn lock_worker( + &self, + binding: super::super::SnapshotLifetimeBinding, + ) -> futures::future::BoxFuture<'_, Result, String>> + { + Box::pin(async move { + let permit = self + .permits + .clone() + .try_acquire_owned() + .map_err(|error| error.to_string())?; + Ok(Box::new(LifetimePermit { + binding, + _permit: permit, + }) as Box) + }) + } fn reserve_worker( &self, mut request: super::super::SnapshotLifetimeRequest, @@ -170,6 +187,338 @@ impl Snapshots for Lifetimes { } } +fn crash_unstarted(owner: SnapshotWorker) -> String { + let id = owner.scope.id().to_owned(); + let store = owner.runtime.inner.store.clone(); + let target = id.clone(); + store.transaction(move |tx| { + tx.execute_batch("CREATE TEMP TABLE preparation_cut(owner TEXT); CREATE TEMP TRIGGER preparation_cut BEFORE INSERT ON event WHEN EXISTS(SELECT 1 FROM preparation_cut WHERE owner=NEW.aggregate_id) BEGIN SELECT RAISE(ABORT,'simulated abrupt preparation death'); END;")?; + tx.execute("INSERT INTO preparation_cut(owner) VALUES (?1)", [target])?; + Ok(()) + }).unwrap(); + drop(owner); // Suppress only Drop's record to model an abrupt process boundary. + store + .transaction(|tx| { + tx.execute_batch("DROP TRIGGER preparation_cut; DROP TABLE preparation_cut;")?; + Ok(()) + }) + .unwrap(); + id +} + +#[tokio::test] +async fn preparation_lost_commit_reply_is_lookup_only_after_reopening_the_real_store() { + fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("lost preparation reply".into()), + )) + } + let (temp, base, _, _, _) = super::super::binary_documents_tests::fixture().await; + let path = temp.path().join("preparation-pair.db"); + let store = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(path.clone()), + Runtime::registry(), + )) + .unwrap(); + let runtime = runtime_with_snapshots( + &base, + Arc::new(store), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + }), + ); + let session = runtime + .create_session(super::super::CreateSession { + directory: temp.path().to_str().unwrap().into(), + model: "test/main".into(), + ..Default::default() + }) + .await + .unwrap() + .id; + let id = crash_unstarted(worker(&runtime, &session).await); + let review = runtime + .snapshot_preparation_review(&session, &id) + .await + .unwrap(); + let confirmation = super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "lost-preparation".into(), + }; + runtime + .inner + .store + .transaction(|tx| { + tx.wal_hook(Some(lost_reply)); + Ok(()) + }) + .unwrap(); + let notifications = runtime.inner.store.subscribe(); + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .is_err() + ); + assert!(notifications.try_recv().is_err()); + let receipt = runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap() + .unwrap(); + runtime.shutdown().await; + drop(runtime); + let reopened = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(path), + Runtime::registry(), + )) + .unwrap(); + let reloaded = runtime_with_store(&base, Arc::new(reopened), temp.path()); + // NoSnapshots cannot reacquire any proof. This must use only the original pair. + assert_eq!( + reloaded + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .unwrap(), + receipt + ); + reloaded.shutdown().await; + base.shutdown().await; +} + +#[tokio::test] +async fn preparation_cancellation_is_paired_and_retry_never_reopens_native_proof() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: permits.clone(), + foreign: false, + }), + ); + let owner = worker(&runtime, &session).await; + let id = owner.scope.id().to_owned(); + assert!( + runtime + .snapshot_preparation_review(&session, &id) + .await + .is_err() + ); + assert_eq!( + runtime.snapshot_preparations(&session).await.unwrap()[0].activity_id, + id + ); + crash_unstarted(owner); + let handle = runtime.inner.handle(&session).await.unwrap(); + handle.mark_snapshot_uncertain(&id); + handle.mark_snapshot_uncertain("op_other"); + legacy_generic_unknown(&runtime, &handle, &session); + let before = runtime.state(&session).await.unwrap().last_seq; + let review = runtime + .snapshot_preparation_review(&session, &id) + .await + .unwrap(); + let confirmation = super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "preparation-one".into(), + }; + let receipt = runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .unwrap(); + assert_eq!(receipt.lifetime, review.lifetime); + assert_eq!(runtime.state(&session).await.unwrap().last_seq, before); + assert!(handle.location_uncertain.load(Ordering::SeqCst)); + assert!(!handle.snapshot_uncertain.lock().unwrap().contains(&id)); + assert!( + handle + .snapshot_uncertain + .lock() + .unwrap() + .contains("op_other") + ); + permits.close(); + assert_eq!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .unwrap(), + receipt + ); + assert_eq!( + runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap(), + Some(receipt) + ); + assert!( + runtime + .snapshot_preparation_confirm(&session, "op_other", &confirmation) + .await + .is_err() + ); + let mut changed = confirmation; + changed.confirmation = "0".repeat(64); + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &changed) + .await + .is_err() + ); + runtime.shutdown().await; + base.shutdown().await; +} + +#[tokio::test] +async fn preparation_writer_failure_rolls_back_both_owners_and_changed_session_refuses_confirmation() + { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: permits.clone(), + foreign: false, + }), + ); + let id = crash_unstarted(worker(&runtime, &session).await); + let review = runtime + .snapshot_preparation_review(&session, &id) + .await + .unwrap(); + let confirmation = super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "preparation-failure".into(), + }; + let notifications = runtime.inner.store.subscribe(); + runtime.inner.store.transaction(|tx| { tx.execute_batch("CREATE TEMP TRIGGER preparation_pair_failure BEFORE INSERT ON event WHEN NEW.type='snapshot.preparation.cancelled.1' BEGIN SELECT RAISE(ABORT,'preparation pair failure'); END;")?; Ok(()) }).unwrap(); + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .is_err() + ); + runtime + .inner + .store + .transaction(|tx| { + tx.execute_batch("DROP TRIGGER preparation_pair_failure;")?; + Ok(()) + }) + .unwrap(); + assert!(notifications.try_recv().is_err()); + assert_eq!(latest(&runtime, &session)["status"], "pending"); + assert!( + runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap() + .is_none() + ); + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + runtime + .inner + .commit_locked( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "later revision".into(), + usage: None, + cost: None, + model: None, + }, + )], + ) + .unwrap(); + drop(state); + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .is_err() + ); + assert_eq!(permits.available_permits(), 1); + runtime.shutdown().await; + base.shutdown().await; +} + +#[tokio::test] +async fn cancelling_an_unstarted_resolution_keeps_its_original_failed_restore_fenced() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + }), + ); + let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + assert!( + runtime + .snapshot_preparation_review(&session, &original.activity_id) + .await + .is_err() + ); + let request = recovery_request(original.clone()); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + drop(state); + let id = crash_unstarted(owner); + let review = runtime + .snapshot_preparation_review(&session, &id) + .await + .unwrap(); + runtime + .snapshot_preparation_confirm( + &session, + &id, + &super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "cancel-resolution".into(), + }, + ) + .await + .unwrap(); + let lookup = runtime + .snapshot_recovery_lookup(&session, &request.confirmation.request_id) + .await + .unwrap() + .unwrap(); + assert_eq!( + lookup.state, + super::super::SnapshotRecoveryState::CancelledBeforeOrigin + ); + let source = original.activity_id.clone(); + let status: String = runtime.inner.store.read(move |db| Ok(db.query_row("SELECT json_extract(data,'$.status') FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", [source], |row| row.get(0))?)).unwrap(); + assert_eq!(status, "unknown"); + let state = handle.state.lock().await; + assert!(runtime.snapshot_worker(&handle, &state).await.is_err()); + drop(state); + runtime.shutdown().await; + base.shutdown().await; +} + #[tokio::test] async fn pre_origin_lifetime_is_bound_before_reservation_and_retained_through_request_disposal() { let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 83d897eca..818fa8017 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -237,6 +237,37 @@ impl GitSnapshots { } impl Snapshots for GitSnapshots { + fn lock_worker( + &self, + binding: cyber_server::runtime::SnapshotLifetimeBinding, + ) -> BoxFuture<'_, Result, String>> { + Box::pin(async move { + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = binding; + Err("Native snapshot lifetime recovery is unavailable".into()) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + let directory = binding + .owner + .directory + .to_str() + .ok_or("Invalid snapshot lifetime Location")?; + self.recovery_location(directory)?; + let directory = directory.to_owned(); + let options = self.recovery_options(&directory); + let data = self.data_dir.clone(); + tokio::task::spawn_blocking(move || { + let store = copy::Store::recover_existing(&data, &directory, options)?; + Ok(Box::new(store.lock_lifetime(&binding)?) + as Box) + }) + .await + .map_err(|error| error.to_string())? + } + }) + } fn reserve_worker( &self, request: cyber_server::runtime::SnapshotLifetimeRequest, diff --git a/docs/rewind.md b/docs/rewind.md index 97d825fba..89f8c540e 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -230,9 +230,49 @@ prove an old worker has exited; older origins without this proof stay inspectabl and fenced. Retained file-copy evidence remains inspectable when capture is disabled or capture limits are lowered. Resolution writes still obey the cap. -The native keep/rollback backend is implemented, but operator confirmation is not -yet exposed: dedicated runtime admission, durable native acknowledgement and -lost-reply reconciliation remain required. Created source directories and private -payloads remain protected for acknowledged safe cleanup. Do not edit records or -remove evidence to force a retry. Complete operator recovery, Git journaling, -reclamation and broader native crash/platform acceptance remain unfinished. +Confirm a reviewed keep/rollback using its exact fingerprint and a retained request ID: + +```sh +cyber api POST /sessions//rewind/recovery/confirm \ + -d '{"journal_id":"restore_...","action":"rollback","confirmation":"","request_id":""}' +cyber api GET /sessions//rewind/recovery/requests/ +``` + +A repeated confirmation looks up that original request, including an incomplete +outcome; it never runs the file changes again. Changed request content refuses. +Lookup does not inspect native files, so `journal_acknowledged: null` means that +response has not observed the journal acknowledgement. For an already committed +recovery receipt, POST its `/acknowledge` child to reconcile only missing journal +acknowledgements. Ordinary successful file-copy rewinds use GET +`/sessions//rewind/restores/` and POST its `/acknowledge` +child. These operations preserve later files, including unsupported replacements, +and do not move the transcript. Historical receipts without immutable proof cannot +authorize new journal writes. + +If a crash occurred before an origin or restore journal was published, list the +retained preparations and review the selected activity: + +```sh +cyber api GET /sessions//rewind/preparations +cyber api POST /sessions//rewind/preparations//review +cyber api POST /sessions//rewind/preparations//confirm \ + -d '{"confirmation":"","request_id":""}' +cyber api GET /sessions//rewind/preparations//requests/ +``` + +The SDK exposes `snapshotPreparations`, `snapshotPreparationReview`, +`snapshotPreparationConfirm` and `snapshotPreparationLookup` on `client.session`. +Review requires an exact retained lifetime lock proving that this worker exited, +a positive unstarted reservation, and no published origin. Confirmation cancels +only that preparation: source files and transcript remain unchanged. Live workers, +foreign or changed evidence, published origins and legacy reservations without +lifetime proof refuse. Checkout/process uncertainty remains independent. Cancelling +an unstarted keep/rollback request leaves its original failed restore unresolved; +that request's lookup reports `cancelled_before_origin`. Retrying a cancellation +uses its original durable receipt without reopening native proof. + +Created source directories, private payloads and lifetime preparations remain +protected for acknowledged safe cleanup. Do not edit records or remove evidence +to force a retry. Origin-before-intent recovery, independent managed-checkout proof, +Git journaling, reclamation, typed recovery clients and broader native crash/platform +acceptance remain unfinished. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 413838d9f..bcb9fe1c3 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1006,3 +1006,91 @@ port separation is retained. No critical/major findings remain for this prerequi its explicitly missing consuming recovery protocol and broader acceptance gates are not accepted by these tests. This is local progress, not a merged delivery or completion of either issue. + + +### Consuming an exited reservation before origin publication + +The owner can now list `/sessions/{id}/rewind/preparations`, POST the selected +`/{activity_id}/review` and `/confirm` operations, and GET its +`/requests/{request_id}` receipt. The matching typed SDK operations are generated. +Review/confirmation require a positive sequence-zero filesystem-only snapshot +reservation, an unchanged exact lifetime binding, no published origin, and at most +its unlaunched unknown-disposal record. Acquiring the original native lifetime lock +proves exit. Missing journal records are never used as evidence. Legacy, generic, +binary, live, foreign, changed or already-launched owners refuse. + +Confirmation retains current recovery authority and Session revision without +reopening closed boundaries. It atomically pairs settlement of that exact native +activity with a separate Session-creation/request-derived cancellation receipt. +The real proof, lifecycle and active owner stay retained through the writer's +COMMIT and its observed reply. Cancellation changes no Session events or source +files, clears only its exact cached snapshot reason, and preserves opaque and other +snapshot uncertainty. Read-only lookup validates both durable owners before clearing +that same cached reason after a lost reply. Changed request content or cross-activity +reuse refuses before native access; repeat confirmation is lookup-only. Receipt +validation bounds native sequences and fingerprints before interpreting persisted +data, and ordinary refusal paths use conflict errors rather than generic failures. + +An unstarted resolution's own cancellation receipt does not acknowledge its failed +parent restore. Recovery-request lookup reports `cancelled_before_origin`, no +resolution origin/receipt and unobserved journal acknowledgement. The parent still +requires its original keep/rollback recovery. Managed checkout/process uncertainty +is neither inferred settled nor cleared by this operation. + +Runtime tests model an abrupt pre-origin cut by rejecting only the scope's disposal +append with a writer-local SQLite trigger. They verify live refusal, exact paired +settlement, independent uncertainty, closed native-port retry, content/cross-owner +refusal and unchanged Session revision. A second trigger rejects insertion of the +request receipt after the original native append, verifying rollback of both owners +and absence of notifications. An actual file-backed Store WAL-hook case commits but +loses the pair reply, reopens that Store under `NoSnapshots`, and finds the original +receipt without native replay. The unstarted resolution test proves the parent +restore stays unknown. These are runtime/Store fixtures, not process-kill proof. + +The real CLI/API test prepares 1280 binary files with shared 64 KiB images, starts a +reviewed rewind, observes its durable reservation at sequence zero, kills the actual +service and verifies no origin was published. After restart, it creates a later +owner symlink, reviews/confirms cancellation, kills/restarts again, and checks the +same receipt by retry and lookup. The symlink/target bytes, another source image, +transcript and inference count are preserved; an explicit subsequent prompt is +admitted. The cut uses actual service death without a restore fault hook. The first +fixture used large text diffs and correctly hit the 4 MiB preview bound; binary +images keep that reviewed preview bounded while retaining the native scan window. + +Origin-before-intent still needs explicit positive source-admission phases and its +own consuming proof. Independent managed-checkout recovery, Git journaling, +reclamation, typed CLI/TUI/App recovery and full platform/cut acceptance remain open. +M2–M8 and both issues are incomplete; this does not make the whole feature merge-ready. + + +Final local macOS arm64 verification: 413 server unit tests passed with one ignored +existing subprocess fixture; all 22 affected integrations passed (4 idle- +cancellation, 13 Location, 5 rewind-preview). All 77 snapshot cases passed (42 unit, +18 file-copy, 17 Git), with its lifetime child fixture listed ignored and invoked +by its real process-kill parents. All 15 real CLI branch/recovery cases passed, +including the new pre-origin service kill and existing partial-exchange cuts. +OpenAPI consistency, SDK generation/typecheck and 133 SDK tests passed; 10 configured +SDK process tests were skipped because `CYBER_BIN` was unset. All-target +server/snapshot/CLI Clippy with warnings denied, formatting and diff checks passed. +Strict pinned OpenSpec passed 117 items; lint reports 0 errors/25 warnings and +inventory stays 53 capabilities/1030 requirements. Sandbox deprecation and Darwin +linker warnings were observed without assigning their origin to main. Linux/Windows, +full-workspace and TUI/App recovery acceptance were not run here. + +The code-review skill was applied across all changed runtime/native/HTTP/test/docs +and generated SDK files. It checks positive unstarted history, immutable native +ownership and current authority, exact paired request identity and lookup, descriptor +retention through COMMIT/reply, cancellation/Stop controls, sequence bounds, error +contracts and separation from restore/binary/generic acknowledgements. Review adds +strict receipt validation and validates the latest native event rather than +filtering past a later foreign event. Native proof verification is explicitly +restricted to synchronous metadata and cannot reenter the runtime/Store from the +writer. The repository runtime-port/native-adapter split is preserved. No critical +or major findings remain for this increment; complete early-resource recovery and +other parent acceptance gaps remain open. + +The initial runtime fixture attempted to use `set_append_fault`, whose documented +scope is ordinary `append`; it does not intercept checked/paired appends. Writer-local +SQLite triggers now model the abrupt disposal and second-owner insertion cuts, +and the final suites above pass. No production Store fault semantics were changed. +This increment remains local and does not complete M2–M8, issue #93 or issue #92. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 0c3e25281..399032499 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -12,7 +12,8 @@ Implementation checkpoint: the non-Git review/confirm/lookup API and SDK now use durable request reservation and paired native settlement. Local real-service partial-restore kills exercise keep/rollback and lookup without replay. Explicit journal acknowledgement reconciliation is implemented for new file-copy receipts; -pre-origin resource cuts, Git journaling, +reserved-before-origin cancellation is available for new lifetime-bound file-copy +workers. Origin-before-intent and independent resource cuts, Git journaling, reclamation, complete clients and the platform matrix remain unfinished. Parent tasks are open; the complete recovery delivery is not merge-ready. @@ -240,3 +241,24 @@ missing journal had no source effects, or acknowledge managed checkout/process uncertainty. The explicit pre-effect consuming protocol, origin-before-intent cut, Git equivalent and complete native/client acceptance remain required. Older reservations without this binding receive no retrospective proof. + + +### Consuming a reservation that never published an origin + +The pre-origin owner path reviews a specific native activity and confirms a +fingerprint before cancelling that preparation. Eligibility requires the positive +sequence-zero snapshot reservation, its unchanged exact lifetime binding, no +historical published origin, and no more than its unlaunched disposal record. +Acquiring the original lifetime file proves that the actual worker has exited; +missing journal files are not used as proof. Historical reservations without the +binding, generic/binary activities, launched origins and live/replaced locks refuse. + +Confirmation preserves all source files and transcript state. A single checked +paired append acknowledges only that original snapshot activity and records the +request receipt in a Session-creation/request-derived owner aggregate. The writer +rechecks the native proof, original record, Session revision and captured recovery +authority. Retry looks up that durable pair before native access and refuses changed +request content or cross-activity reuse. Cancelling an unstarted recovery request +leaves its original failed restore fenced. Checkout/process uncertainty remains +independent; no Session lease is acknowledged by this operation. This path does +not resolve the origin-before-intent cut or admit a rollback without source proof. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 419f32032..e2ea800a1 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -99,6 +99,12 @@ - **AND** replacement, missing proof or a live lock refuses reacquisition; a privately staged incomplete publication remains retained and cannot grant acknowledgement authority - **AND** this evidence counts toward the Location cap, protects pending references from GC, and cannot settle a generic process, Session-owned checkout lease or historical reservation without that binding +#### Scenario: Owner cancels an exited preparation with no published origin +- **WHEN** the owner confirms a fingerprint review of a positive unstarted reservation under its exact reacquired native lifetime lock +- **THEN** the writer atomically acknowledges only that activity and retains its cancellation request receipt, rechecking current Session/recovery authority without moving source files or transcript state +- **AND** retry looks up the original durable pair without native access, changed or cross-owner request reuse refuses, and a lost commit reply does not dispatch another operation +- **AND** live, legacy, generic, binary or already-launched owners refuse; independent checkout/process uncertainty remains fenced, and cancelling an unstarted resolution leaves its failed parent restore unresolved + #### Scenario: Session changes before a source effect - **WHEN** the Session revision changes after preparation but before the next source mutation - **THEN** that mutation is refused and existing owner bytes remain unchanged diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 9449c182a..f568e8d1d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -90,9 +90,10 @@ The actual worker keeps it through resource teardown, and source authorization revalidates its native route. Lock metadata and orphan stages count toward the cap and conservatively protect GC. Native tests reacquire only after actual process death and preserve an interrupted pre-publication stage. This is a prerequisite, -not completion of pre-origin recovery: its consuming operator path, positive -source-admission phases, independent managed-checkout proof and Git equivalent -remain open. No new acknowledgement admission or force-clear is enabled. +not completion of all pre-origin recovery. The following continuation adds the +consuming path only for a positively bound reservation with no published origin. +Positive source-admission phases for origin-before-intent, independent managed- +checkout proof and Git equivalence remain open; no force-clear is enabled. Implementation source map: @@ -151,3 +152,13 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - [ ] 7.2 Run affected Rust tests, format/Clippy, SDK typecheck/tests, TUI/real-binary acceptance, strict OpenSpec/lint and relevant native/platform CI. Record skips and unperformed external checks explicitly. - [ ] 7.3 Use the code-review skill for substantive review, address findings and deliver reviewable PRs. Merge only when authorized; archive after accepted merge and overlap reconciliation. - [ ] 7.4 Verify all approved first- and second-workstream outcomes against evidence before marking the combined goal complete. + + +Reserved-before-origin continuation: list/review/confirm/lookup operations now +consume the exact exited worker lifetime proof and atomically pair only that native +acknowledgement with a retained cancellation request. A real service kill before +origin publication preserves later symlink/owner edits and the transcript, supports +retry after restart and admits a subsequent prompt in an unmanaged non-Git Location. +Cancelling an unstarted resolution reports `cancelled_before_origin` without +acknowledging its original failed restore. Origin-before-intent, independent checkout +resources, complete clients, Git, reclamation and the remaining M8 cuts remain open. diff --git a/sdk/openapi.json b/sdk/openapi.json index ec4da20fa..055e2754d 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7440,7 +7440,8 @@ "pending", "unknown", "acknowledged", - "resolved_by_another_request" + "resolved_by_another_request", + "cancelled_before_origin" ] }, "SnapshotRecoveryReceipt": { @@ -7527,6 +7528,208 @@ ], "description": "Exact native intent and Session batch consumed by an ordinary restore.\nThis is durable acknowledgement evidence, never authority to replay files." }, + "Data_Array_of_SnapshotPreparation": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SnapshotPreparation" + } + } + }, + "required": [ + "data" + ] + }, + "SnapshotPreparation": { + "type": "object", + "properties": { + "activity_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "lifetime_available": { + "type": "boolean" + } + }, + "required": [ + "activity_id", + "status", + "lifetime_available" + ] + }, + "Data_SnapshotPreparationReview": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/SnapshotPreparationReview" + } + }, + "required": [ + "data" + ] + }, + "SnapshotPreparationReview": { + "type": "object", + "properties": { + "activity_id": { + "type": "string" + }, + "lifetime": { + "$ref": "#/components/schemas/SnapshotLifetimeBinding" + }, + "native_sequence": { + "type": "integer", + "format": "int64" + }, + "session_sequence": { + "type": "integer", + "format": "int64" + }, + "confirmation": { + "type": "string" + }, + "warning": { + "type": "string" + } + }, + "required": [ + "activity_id", + "lifetime", + "native_sequence", + "session_sequence", + "confirmation", + "warning" + ] + }, + "SnapshotLifetimeBinding": { + "type": "object", + "additionalProperties": false, + "properties": { + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "owner": { + "$ref": "#/components/schemas/SnapshotLifetimeRequest" + }, + "root_identity": { + "type": "string" + }, + "namespace_identity": { + "type": "string" + }, + "directory_identity": { + "type": "string" + }, + "file_identity": { + "type": "string" + } + }, + "required": [ + "version", + "owner", + "root_identity", + "namespace_identity", + "directory_identity", + "file_identity" + ] + }, + "SnapshotLifetimeRequest": { + "type": "object", + "additionalProperties": false, + "properties": { + "activity_id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "session_created_id": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "worktree_id": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "activity_id", + "session_id", + "session_created_id", + "directory" + ] + }, + "SnapshotPreparationConfirmation": { + "type": "object", + "additionalProperties": false, + "properties": { + "confirmation": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "confirmation", + "request_id" + ] + }, + "Data_SnapshotPreparationReceipt": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/SnapshotPreparationReceipt" + } + }, + "required": [ + "data" + ] + }, + "SnapshotPreparationReceipt": { + "type": "object", + "additionalProperties": false, + "properties": { + "activity_id": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "confirmation": { + "type": "string" + }, + "lifetime": { + "$ref": "#/components/schemas/SnapshotLifetimeBinding" + }, + "native_sequence": { + "type": "integer", + "format": "int64" + }, + "session_sequence": { + "type": "integer", + "format": "int64" + } + }, + "required": [ + "activity_id", + "request_id", + "confirmation", + "lifetime", + "native_sequence", + "session_sequence" + ] + }, "RewindSelection": { "type": "object", "additionalProperties": false, @@ -24660,6 +24863,228 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/preparations": { + "get": { + "operationId": "v1.session.snapshotPreparations", + "summary": "List retained snapshot reservations without a published origin or native access", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_Array_of_SnapshotPreparation" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, + "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/review": { + "post": { + "operationId": "v1.session.snapshotPreparationReview", + "summary": "Review cancellation of an exited preparation using its exact native lifetime proof", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "activityID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotPreparationReview" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, + "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/confirm": { + "post": { + "operationId": "v1.session.snapshotPreparationConfirm", + "summary": "Cancel only a fingerprint-confirmed unstarted snapshot preparation without moving files or transcript", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "activityID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotPreparationReceipt" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SnapshotPreparationConfirmation" + } + } + } + } + } + }, + "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}": { + "get": { + "operationId": "v1.session.snapshotPreparationLookup", + "summary": "Look up the original durable preparation cancellation pair without native access or replay", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "activityID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "requestID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotPreparationReceipt" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge": { "post": { "operationId": "v1.session.snapshotRestoreAcknowledge", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index a1e6abbdb..da5997ca0 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1553,6 +1553,30 @@ export interface Operations { body: undefined; response: S.Data_Nullable_Goal; }; + "v1.session.snapshotPreparationConfirm": { + path: { sessionID: string; activityID: string }; + query: Record; + body: S.SnapshotPreparationConfirmation; + response: S.SnapshotPreparationReceipt; + }; + "v1.session.snapshotPreparationLookup": { + path: { sessionID: string; activityID: string; requestID: string }; + query: Record; + body: undefined; + response: S.SnapshotPreparationReceipt; + }; + "v1.session.snapshotPreparationReview": { + path: { sessionID: string; activityID: string }; + query: Record; + body: undefined; + response: S.SnapshotPreparationReview; + }; + "v1.session.snapshotPreparations": { + path: { sessionID: string }; + query: Record; + body: undefined; + response: S.SnapshotPreparation[]; + }; "v1.session.snapshotRecovery": { path: { sessionID: string }; query: Record; @@ -4668,6 +4692,54 @@ export const operations = { unwrap: false, stream: false, }, + "v1.session.snapshotPreparationConfirm": { + tag: "session", + name: "snapshotPreparationConfirm", + method: "POST", + path: "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/confirm", + pathParams: ["sessionID", "activityID"], + query: [], + body: "required", + located: false, + unwrap: true, + stream: false, + }, + "v1.session.snapshotPreparationLookup": { + tag: "session", + name: "snapshotPreparationLookup", + method: "GET", + path: "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}", + pathParams: ["sessionID", "activityID", "requestID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, + "v1.session.snapshotPreparationReview": { + tag: "session", + name: "snapshotPreparationReview", + method: "POST", + path: "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/review", + pathParams: ["sessionID", "activityID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, + "v1.session.snapshotPreparations": { + tag: "session", + name: "snapshotPreparations", + method: "GET", + path: "/api/v1/sessions/{sessionID}/rewind/preparations", + pathParams: ["sessionID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, "v1.session.snapshotRecovery": { tag: "session", name: "snapshotRecovery", @@ -6142,6 +6214,18 @@ export function createGroups(caller: Caller) { /** Clear the current Goal and activate the next queued one; null when the queue was empty (`POST /api/v1/sessions/{sessionID}/goals/skip`) */ skipGoal: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.skipGoal", { path: { sessionID } }, options), + /** Cancel only a fingerprint-confirmed unstarted snapshot preparation without moving files or transcript (`POST /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/confirm`) */ + snapshotPreparationConfirm: (sessionID: string, activityID: string, body: Operations["v1.session.snapshotPreparationConfirm"]["body"], options?: RequestOptions): Promise => + caller.call("v1.session.snapshotPreparationConfirm", { path: { sessionID, activityID }, body }, options), + /** Look up the original durable preparation cancellation pair without native access or replay (`GET /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}`) */ + snapshotPreparationLookup: (sessionID: string, activityID: string, requestID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotPreparationLookup", { path: { sessionID, activityID, requestID } }, options), + /** Review cancellation of an exited preparation using its exact native lifetime proof (`POST /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/review`) */ + snapshotPreparationReview: (sessionID: string, activityID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotPreparationReview", { path: { sessionID, activityID } }, options), + /** List retained snapshot reservations without a published origin or native access (`GET /api/v1/sessions/{sessionID}/rewind/preparations`) */ + snapshotPreparations: (sessionID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotPreparations", { path: { sessionID } }, options), /** Inspect retained filesystem restore journals without acknowledging or replaying native effects (`GET /api/v1/sessions/{sessionID}/rewind/recovery`) */ snapshotRecovery: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecovery", { path: { sessionID } }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index b2734420d..58b7f036f 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -902,6 +902,10 @@ export type Data_Array_of_RemoteReach = { data: RemoteReach[]; }; +export type Data_Array_of_SnapshotPreparation = { + data: SnapshotPreparation[]; +}; + export type Data_Array_of_SnapshotRecovery = { data: SnapshotRecovery[]; }; @@ -1106,6 +1110,14 @@ export type Data_SideAnswer = { data: SideAnswer; }; +export type Data_SnapshotPreparationReceipt = { + data: SnapshotPreparationReceipt; +}; + +export type Data_SnapshotPreparationReview = { + data: SnapshotPreparationReview; +}; + export type Data_SnapshotRecoveryResult = { data: SnapshotRecoveryResult; }; @@ -4707,6 +4719,23 @@ export type SnapshotBinding = { lock_identity?: string | null; }; +export type SnapshotLifetimeBinding = { + version: number; + owner: SnapshotLifetimeRequest; + root_identity: string; + namespace_identity: string; + directory_identity: string; + file_identity: string; +}; + +export type SnapshotLifetimeRequest = { + activity_id: string; + session_id: string; + session_created_id: string; + directory: string; + worktree_id?: string | null; +}; + export type SnapshotOrigin = { version: number; session_id: string; @@ -4716,6 +4745,35 @@ export type SnapshotOrigin = { binding: SnapshotBinding; }; +export type SnapshotPreparation = { + activity_id: string; + status: string; + lifetime_available: boolean; +}; + +export type SnapshotPreparationConfirmation = { + confirmation: string; + request_id: string; +}; + +export type SnapshotPreparationReceipt = { + activity_id: string; + request_id: string; + confirmation: string; + lifetime: SnapshotLifetimeBinding; + native_sequence: number; + session_sequence: number; +}; + +export type SnapshotPreparationReview = { + activity_id: string; + lifetime: SnapshotLifetimeBinding; + native_sequence: number; + session_sequence: number; + confirmation: string; + warning: string; +}; + /** Retained filesystem evidence; completion does not acknowledge a Session/native receipt. */ export type SnapshotRecovery = { id: string; @@ -4807,7 +4865,7 @@ export type SnapshotRecoverySelection = { action: SnapshotRecoveryAction; }; -export type SnapshotRecoveryState = "pending" | "unknown" | "acknowledged" | "resolved_by_another_request"; +export type SnapshotRecoveryState = "pending" | "unknown" | "acknowledged" | "resolved_by_another_request" | "cancelled_before_origin"; /** * Exact native intent and Session batch consumed by an ordinary restore. diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index a109ac7d1..13882edeb 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -71,3 +71,21 @@ test("ordinary restore lookup and acknowledgement preserve immutable paired rece assert.equal(calls[1]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/restores/op_ordinary/acknowledge"); assert.ok(!calls[1]!.body); }); + +test("preparation recovery lists, reviews, confirms and looks up an exact cancellation request", async () => { + const confirmation = { confirmation: "c".repeat(64), request_id: "cancel/with space" }; + const receipt = { activity_id: "op_fixture", request_id: confirmation.request_id, confirmation: confirmation.confirmation }; + const { client, calls } = mockClient(req => json(200, { data: req.url.pathname.endsWith("/preparations") + ? [{ activity_id: "op_fixture", status: "pending", lifetime_available: true }] + : req.url.pathname.endsWith("/review") ? { ...receipt, warning: "Checkout uncertainty remains independent" } : receipt })); + const list = await client.session.snapshotPreparations("ses_owner"); + assert.equal(list[0]!.lifetime_available, true); + const review = await client.session.snapshotPreparationReview("ses_owner", "op_fixture"); + assert.equal(review.confirmation, confirmation.confirmation); + assert.deepEqual(await client.session.snapshotPreparationConfirm("ses_owner", "op_fixture", confirmation), receipt); + assert.deepEqual(await client.session.snapshotPreparationLookup("ses_owner", "op_fixture", confirmation.request_id), receipt); + assert.deepEqual(calls.map(call => call.method), ["GET", "POST", "POST", "GET"]); + assert.ok(!calls[1]!.body); + assert.deepEqual(JSON.parse(calls[2]!.body!), confirmation); + assert.equal(calls[3]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/preparations/op_fixture/requests/cancel%2Fwith%20space"); +}); From 4bdb3f64d300ac2ef11766091e176b73f178ddd5 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 22:09:49 -0400 Subject: [PATCH 12/87] Cancel exited snapshot preparations before source admission --- crates/cyber-cli/tests/branches_cli.rs | 65 ++++- crates/cyber-server/src/http/openapi.rs | 6 +- crates/cyber-server/src/http/sessions.rs | 15 + crates/cyber-server/src/runtime/activity.rs | 89 +++++- .../src/runtime/snapshot_lifetime.rs | 30 +- .../src/runtime/snapshot_preparation.rs | 242 +++++++++++++--- .../src/runtime/snapshot_recovery.rs | 6 +- .../src/runtime/snapshot_recovery_record.rs | 88 ++++++ .../src/runtime/snapshot_recovery_types.rs | 1 + .../cyber-server/src/runtime/snapshot_work.rs | 31 ++ .../src/runtime/snapshot_work_tests.rs | 203 ++++++++++++- crates/cyber-snapshot/src/copy/lifetime.rs | 252 +++++++++++++++- crates/cyber-snapshot/src/copy/mod.rs | 1 + crates/cyber-snapshot/src/copy/preparation.rs | 269 ++++++++++++++++++ crates/cyber-snapshot/src/copy/restore.rs | 26 ++ ...261106010000_snapshot_source_admission.sql | 4 + crates/cyber-store/src/migrate.rs | 46 ++- docs/goal-delivery.md | 9 + docs/rewind.md | 43 +-- .../implementation.md | 81 ++++++ .../managed-recovery.md | 62 ++-- .../specs/snapshots-checkpoints/spec.md | 12 +- .../complete-daily-coding-surfaces/tasks.md | 15 + sdk/openapi.json | 104 ++++++- sdk/typescript/src/generated/operations.ts | 25 +- sdk/typescript/src/generated/types.ts | 6 +- sdk/typescript/test/rewind.test.ts | 11 + 27 files changed, 1623 insertions(+), 119 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/preparation.rs create mode 100644 crates/cyber-store/migrations/20261106010000_snapshot_source_admission.sql diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index fbab42c03..6f3b7af41 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -795,6 +795,17 @@ fn managed_keep_and_rollback_after_real_partial_restore_kill_preserve_owner_file #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_restart() { + cancelled_pre_effect_preparation(false); +} + +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn exited_origin_preparation_before_source_admission_is_cancelled_after_real_kill() { + cancelled_pre_effect_preparation(true); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn cancelled_pre_effect_preparation(published: bool) { let model = scripted_model::ScriptedModel::start(reply); let env = Env::new(&model); let repo = env.root.join("repo"); @@ -811,7 +822,7 @@ fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_res } env.exec(Some(&id), "after snapshot"); let messages = env.messages(&id); - let activity = kill_before_origin(&env, &id, &first); + let activity = kill_before_source_admission(&env, &id, &first, published); let requests = model.requests().len(); let target = env.root.join("preparation-owner-target"); std::fs::write(&target, "later owner bytes").unwrap(); @@ -830,6 +841,14 @@ fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_res let receipt = json(&env.cyber(&["api", "POST", &format!("{route}/confirm"), "-d", &body]))["data"] .clone(); + let acknowledge = format!("{route}/requests/before-origin/acknowledge"); + assert_eq!( + json(&env.cyber(&["api", "POST", &acknowledge]))["data"], + receipt + ); + if published { + assert_preparation_tombstone(&env, &receipt); + } kill_registered_service(&env); let repeated = json(&env.cyber(&["api", "POST", &format!("{route}/confirm"), "-d", &body]))["data"] @@ -839,6 +858,10 @@ fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_res json(&env.cyber(&["api", "GET", &format!("{route}/requests/before-origin")]))["data"] .clone(); assert_eq!(lookup, receipt); + assert_eq!( + json(&env.cyber(&["api", "POST", &acknowledge]))["data"], + receipt + ); assert!( std::fs::symlink_metadata(repo.join("bulk_0000")) .unwrap() @@ -856,7 +879,30 @@ fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_res } #[cfg(any(target_os = "macos", target_os = "linux"))] -fn kill_before_origin(env: &Env, id: &str, first: &str) -> String { +fn assert_preparation_tombstone(env: &Env, receipt: &Value) { + let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let journal = namespace + .join("journals") + .join(receipt["origin"]["binding"]["journal_id"].as_str().unwrap()); + let tombstone: Value = + serde_json::from_slice(&std::fs::read(journal.join("preparation.json")).unwrap()).unwrap(); + assert_eq!(&tombstone, receipt); + assert_eq!(receipt["source_admitted"], false); + if let Ok(bytes) = std::fs::read(journal.join("intent.json")) { + let intent: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(intent["completed"], false); + assert_eq!(intent["acknowledged"], false); + assert!(intent["applied"].as_array().unwrap().is_empty()); + } +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn kill_before_source_admission(env: &Env, id: &str, first: &str, published: bool) -> String { use rusqlite::OptionalExtension; let selection = reviewed_selection(env, first, "both", "commit"); let preview = reviewed_preview(env, id, &selection); @@ -879,7 +925,7 @@ fn kill_before_origin(env: &Env, id: &str, first: &str) -> String { .unwrap(); let deadline = std::time::Instant::now() + Duration::from_secs(30); let activity = loop { - let found: Option = db.query_row("SELECT aggregate_id FROM event e WHERE type='native.activity.changed.1' AND seq=0 AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.snapshot_origin') IS NULL AND json_extract(data,'$.snapshot_lifetime') IS NOT NULL AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq)", [id], |row| row.get(0)).optional().unwrap(); + let found: Option = db.query_row("SELECT aggregate_id FROM event e WHERE type='native.activity.changed.1' AND ((?2=1 AND seq=1 AND json_extract(data,'$.snapshot_origin') IS NOT NULL) OR (?2=0 AND seq=0 AND json_extract(data,'$.snapshot_origin') IS NULL)) AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.snapshot_source_admitted')=0 AND json_extract(data,'$.snapshot_lifetime') IS NOT NULL AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq)", rusqlite::params![id, i64::from(published)], |row| row.get(0)).optional().unwrap(); if let Some(found) = found { break found; } @@ -903,9 +949,14 @@ fn kill_before_origin(env: &Env, id: &str, first: &str) -> String { ) .unwrap(); let record: Value = serde_json::from_str(&data).unwrap(); - assert!( - record["snapshot_origin"].is_null(), - "kill missed the pre-origin cut" + assert_eq!( + record["snapshot_origin"].is_object(), + published, + "kill missed the selected origin boundary" + ); + assert_eq!( + record["snapshot_source_admitted"], false, + "kill missed the pre-effect boundary" ); activity } @@ -1332,7 +1383,7 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_snapshot_lifetime(namespace: &std::path::Path, activity: &Value, origin: &Value) { let lifetime = &activity["snapshot_lifetime"]; - assert_eq!(lifetime["version"], 1); + assert_eq!(lifetime["version"], 2); assert_eq!(lifetime["owner"]["activity_id"], origin["activity_id"]); assert_eq!( lifetime["owner"]["session_created_id"], diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index 845a30a5d..4d84afc4a 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -772,14 +772,16 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { .ok::>(g), op("get", "/sessions/{sessionID}/rewind/restores/{activityID}", "v1.session.snapshotRestoreLookup", "Read the exact committed ordinary restore receipt without native access or replay") .ok::>(g), - op("get", "/sessions/{sessionID}/rewind/preparations", "v1.session.snapshotPreparations", "List retained snapshot reservations without a published origin or native access") + op("get", "/sessions/{sessionID}/rewind/preparations", "v1.session.snapshotPreparations", "List retained preparation candidates without native access; review checks eligibility") .ok::>>(g), op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/review", "v1.session.snapshotPreparationReview", "Review cancellation of an exited preparation using its exact native lifetime proof") .ok::>(g), - op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/confirm", "v1.session.snapshotPreparationConfirm", "Cancel only a fingerprint-confirmed unstarted snapshot preparation without moving files or transcript") + op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/confirm", "v1.session.snapshotPreparationConfirm", "Cancel only a fingerprint-confirmed pre-effect snapshot preparation without moving files or transcript") .body::(g).ok::>(g), op("get", "/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}", "v1.session.snapshotPreparationLookup", "Look up the original durable preparation cancellation pair without native access or replay") .ok::>(g), + op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}/acknowledge", "v1.session.snapshotPreparationAcknowledge", "Reconcile native metadata acknowledgement of an exact committed pre-effect cancellation without source changes") + .ok::>(g), op("post", "/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge", "v1.session.snapshotRestoreAcknowledge", "Acknowledge the immutable ordinary restore intent after its paired Session commit without replaying source files") .ok::>(g), op("post", "/sessions/{sessionID}/rewind/preview", "v1.session.rewindPreview", "Review a rewind without changing files or the active conversation") diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index 656abe13c..10761caa4 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -386,6 +386,10 @@ pub fn routes() -> Router { &format!("{s}/rewind/preparations/{{activity_id}}/requests/{{request_id}}"), get(snapshot_preparation_lookup), ) + .route( + &format!("{s}/rewind/preparations/{{activity_id}}/requests/{{request_id}}/acknowledge"), + post(snapshot_preparation_acknowledge), + ) .route(&format!("{s}/rewind/confirm"), post(rewind_confirm)) .route(&format!("{s}/revert/stage"), post(stage)) .route(&format!("{s}/revert/clear"), post(clear)) @@ -1365,6 +1369,17 @@ async fn snapshot_preparation_lookup( })?, })) } +async fn snapshot_preparation_acknowledge( + State(state): State, + Path((id, activity, request)): Path<(String, String, String)>, +) -> Result>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_preparation_acknowledge(&id, &activity, &request) + .await?, + })) +} async fn rewind_preview( State(state): State, Path(id): Path, diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 81bea18ea..34f49e488 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -40,6 +40,8 @@ pub(super) struct Record { #[serde(default, skip_serializing_if = "Option::is_none")] pub snapshot_preparation_receipt: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_source_admitted: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_bindings: Option>, } pub(super) struct Scope { @@ -52,6 +54,7 @@ pub(super) struct Scope { pub control: Arc, launched: bool, settled: bool, + source_admitted: std::sync::Mutex, } enum Authority { Ordinary(AdmissionAuthority), @@ -134,6 +137,7 @@ impl Scope { snapshot_request: Some(request.clone()), snapshot_receipt: None, snapshot_restore_receipt: None, + snapshot_source_admitted: lifetime.as_ref().map(|_| false), snapshot_lifetime: lifetime, snapshot_preparation_receipt: None, admission_bindings: Some(authority.bindings().to_vec()), @@ -177,6 +181,7 @@ impl Scope { control, launched: false, settled: false, + source_admitted: std::sync::Mutex::new(false), }) } async fn reserve_owner( @@ -202,6 +207,7 @@ impl Scope { snapshot_request: None, snapshot_receipt: None, snapshot_restore_receipt: None, + snapshot_source_admitted: lifetime.as_ref().map(|_| false), snapshot_lifetime: lifetime, snapshot_preparation_receipt: None, admission_bindings: Some(authority.bindings.clone()), @@ -232,6 +238,7 @@ impl Scope { control, launched: false, settled: false, + source_admitted: std::sync::Mutex::new(false), }) } pub fn launch(&mut self) -> Result<(), RuntimeError> { @@ -271,6 +278,63 @@ impl Scope { .then_some(self.record.snapshot_origin.as_ref()) .flatten() } + fn sequence(&self) -> i64 { + if *self + .source_admitted + .lock() + .unwrap_or_else(PoisonError::into_inner) + { + 2 + } else if self.launched { + 1 + } else { + 0 + } + } + fn snapshot_record(&self) -> Record { + let mut record = self.record.clone(); + if record.snapshot_source_admitted.is_some() { + let admitted = *self + .source_admitted + .lock() + .unwrap_or_else(PoisonError::into_inner); + record.snapshot_source_admitted = Some(admitted); + if admitted { + record.phase = "snapshot_source_admitted".into(); + } + } + record + } + pub(super) fn snapshot_lifetime(&self) -> Option<&super::SnapshotLifetimeBinding> { + self.record.snapshot_lifetime.as_ref() + } + /// A committed marker precedes the first source mutation. A lost reply stops + /// effects and conservatively retains a possibly-admitted outcome. + pub(super) fn admit_snapshot_source(&self) -> Result<(), RuntimeError> { + if self.record.snapshot_source_admitted.is_none() { + return Ok(()); + } + let mut admitted = self + .source_admitted + .lock() + .unwrap_or_else(PoisonError::into_inner); + if *admitted { + return Ok(()); + } + self.verify_authority()?; + if self.snapshot_origin().is_none() { + return Err(RuntimeError::Conflict( + "Source admission lacks its native origin".into(), + )); + } + let mut record = self.record.clone(); + record.phase = "snapshot_source_admitted".into(); + record.snapshot_source_admitted = Some(true); + self.store + .append(&self.id, Expected::Seq(1), vec![change(&record)])?; + *admitted = true; + Ok(()) + } pub(super) fn commit_snapshot( &mut self, state: &mut super::SessionState, @@ -292,7 +356,7 @@ impl Scope { "Snapshot commit authority changed".into(), )); } - let mut settled = self.record.clone(); + let mut settled = self.snapshot_record(); settled.status = "settled".into(); settled.phase = "snapshot_restore_committed".into(); settled.admission_bindings = None; @@ -310,7 +374,7 @@ impl Scope { let acknowledgement = change(&settled); let owners = [ (state.info.id.clone(), state.last_seq), - (self.id.clone(), if self.launched { 1 } else { 0 }), + (self.id.clone(), self.sequence()), ]; let (stored, ()) = self.store.append_pair_checked(owners, move |tx| { if !super::admission_authority::bindings_current(tx, &bindings)? { @@ -334,10 +398,12 @@ impl Scope { "Recovery requires its paired native acknowledgement".into(), )); } - self.record.status = "settled".into(); - self.record.admission_bindings = None; + let mut record = self.snapshot_record(); + record.status = "settled".into(); + record.admission_bindings = None; self.store - .append(&self.id, Expected::Any, vec![change(&self.record)])?; + .append(&self.id, Expected::Any, vec![change(&record)])?; + self.record = record; self.settled = true; Ok(()) } @@ -388,7 +454,7 @@ impl Scope { .store .aggregate_seq(&original.activity_id)? .ok_or_else(|| RuntimeError::Conflict("Original native owner disappeared".into()))?; - let mut settled = self.record.clone(); + let mut settled = self.snapshot_record(); settled.status = "settled".into(); settled.phase = "snapshot_recovery_committed".into(); settled.admission_bindings = None; @@ -400,7 +466,7 @@ impl Scope { let stop = self.control.stop.clone(); let owners = [ (original.activity_id.clone(), original_sequence), - (self.id.clone(), 1), + (self.id.clone(), self.sequence()), ]; self.store.append_pair_checked(owners, move |tx| { let (_, mut original_record) = super::snapshot_recovery_record::attest(tx, &original)?; @@ -457,7 +523,7 @@ impl Scope { let id = self.id.clone(); let original = self.record.clone(); let handle = self.handle.clone(); - let sequence = if self.launched { 1 } else { 0 }; + let sequence = self.sequence(); // Publish before enqueueing. Even an unavailable writer retains the exact // owner; a later writer acknowledgement can clear it without racing a // post-commit caller-side flag write. @@ -477,7 +543,7 @@ impl Scope { cyber_store::StoreError::CorruptEvent { id: id.clone(), reason: error.to_string() })?; // An unsuccessful launch may have set the in-memory origin without // publishing it. Its durable reservation must still have no origin. - let origin = if sequence == 1 { original.snapshot_origin.as_ref() } else { None }; + let origin = if original.snapshot_origin.is_some() && sequence >= 1 { original.snapshot_origin.as_ref() } else { None }; if kind != CHANGED || current.session_id != original.session_id || current.binary_document != original.binary_document || current.snapshot_worker != original.snapshot_worker @@ -495,6 +561,9 @@ impl Scope { } return Ok((vec![], false)); } + if current.snapshot_source_admitted != record.snapshot_source_admitted { + return Err(cyber_store::StoreError::InvalidAppend("Snapshot source-admission state changed before disposal".into())); + } if seq != sequence { return Err(cyber_store::StoreError::Concurrency { aggregate: id, expected: sequence, actual: seq }); } @@ -512,7 +581,7 @@ impl Drop for Scope { // This trusted scope owns only synchronous filesystem calls. Once // its actual worker unwinds, it has no escaped task/process to settle; // possible file effects remain in the separate document intent. - let mut disposed = self.record.clone(); + let mut disposed = self.snapshot_record(); disposed.snapshot_origin = self.snapshot_origin().cloned(); disposed.status = if (self.launched || self.record.snapshot_request.is_some()) && !filesystem_only { diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs index 2482a4aeb..9605f1b62 100644 --- a/crates/cyber-server/src/runtime/snapshot_lifetime.rs +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -21,6 +21,8 @@ pub struct SnapshotLifetimeBinding { pub namespace_identity: String, pub directory_identity: String, pub file_identity: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_phase: Option, } /// The actual worker retains this lock through all resource acquisition and work. @@ -30,6 +32,25 @@ pub trait SnapshotLifetime: Send { /// Synchronous native metadata verification; may run inside the Store writer. /// It must not call the runtime/Store, launch processes or perform source effects. fn verify(&self) -> Result<(), String>; + /// Persist native admission before a source callback. A legacy lifetime may + /// never grant the new positive pre-effect cancellation proof. + fn admit_source(&self, _origin: &super::SnapshotOrigin) -> Result<(), String> { + Err("Native source admission is unavailable".into()) + } + /// Positive native source-phase proof, not merely an unlocked lifetime file. + fn verify_preparation(&self, _origin: Option<&super::SnapshotOrigin>) -> Result<(), String> { + Err("Native preparation phase proof is unavailable".into()) + } + /// Metadata publication only, after observed durable preparation cancellation. + fn acknowledge_preparation( + &mut self, + receipt: &super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + if receipt.origin.is_some() { + return Err("Preparation journal acknowledgement is unavailable".into()); + } + Ok(()) + } } impl super::Runtime { @@ -66,7 +87,14 @@ impl super::Runtime { .map_err(super::RuntimeError::Conflict)?; if let Some(lifetime) = &lifetime { let binding = lifetime.binding(); - if binding.version != 1 + if binding.version != 2 + || binding.source_phase.as_deref().is_none_or(|phase| { + !phase.starts_with("phase_") + || phase.len() > 128 + || !phase + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + }) || binding.owner != request || [ &binding.root_identity, diff --git a/crates/cyber-server/src/runtime/snapshot_preparation.rs b/crates/cyber-server/src/runtime/snapshot_preparation.rs index 3af319648..65c3efbad 100644 --- a/crates/cyber-server/src/runtime/snapshot_preparation.rs +++ b/crates/cyber-server/src/runtime/snapshot_preparation.rs @@ -1,4 +1,4 @@ -//! Consuming proof for an exited snapshot worker that never published an origin. +//! Consuming proof for an exited snapshot worker with no admitted source effects. use super::snapshot_recovery_authority::RecoveryAuthority; use super::{Runtime, RuntimeError, SnapshotLifetime, SnapshotLifetimeBinding, activity}; use cyber_store::{NewEvent, StoreError}; @@ -29,6 +29,7 @@ pub struct SnapshotPreparationReview { pub session_sequence: i64, pub confirmation: String, pub warning: String, + pub origin: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] @@ -39,6 +40,14 @@ pub struct SnapshotPreparationReceipt { pub lifetime: SnapshotLifetimeBinding, pub native_sequence: i64, pub session_sequence: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_admitted: Option, +} + +impl SnapshotPreparationReceipt { + pub const MAX_ENCODED_BYTES: u64 = 64 * 1024; } struct Target { @@ -65,9 +74,9 @@ pub(super) fn validate_event(data: &serde_json::Value) -> Result<(), String> { validate_receipt(&receipt).map_err(|error| error.to_string()) } fn validate_receipt(receipt: &SnapshotPreparationReceipt) -> Result<(), StoreError> { - if !(0..=1).contains(&receipt.native_sequence) + if !(0..=2).contains(&receipt.native_sequence) || receipt.session_sequence < 0 - || receipt.lifetime.version != 1 + || !matches!(receipt.lifetime.version, 1 | 2) || receipt.lifetime.owner.activity_id != receipt.activity_id || receipt.request_id.is_empty() || receipt.request_id.len() > 256 @@ -80,6 +89,26 @@ fn validate_receipt(receipt: &SnapshotPreparationReceipt) -> Result<(), StoreErr { return Err(invalid("Invalid preparation cancellation receipt")); } + match &receipt.origin { + None if receipt.native_sequence <= 1 && receipt.source_admitted != Some(true) => {} + Some(origin) + if receipt.source_admitted == Some(false) + && receipt.lifetime.version == 2 + && receipt.lifetime.source_phase.is_some() + && receipt.native_sequence >= 1 + && origin.version == 1 + && origin.activity_id == receipt.activity_id + && origin.session_id == receipt.lifetime.owner.session_id + && origin.session_created_id == receipt.lifetime.owner.session_created_id + && origin.worktree_id == receipt.lifetime.owner.worktree_id + && origin.binding.directory == receipt.lifetime.owner.directory + && origin.binding.root_identity == receipt.lifetime.root_identity => {} + _ => { + return Err(invalid( + "Cancellation receipt has no positive pre-effect proof", + )); + } + } Ok(()) } fn target(db: &Connection, session: &str, activity: &str) -> Result { @@ -98,21 +127,19 @@ fn target(db: &Connection, session: &str, activity: &str) -> Result Result {} + Some(origin) + if reserved.snapshot_source_admitted == Some(false) + && record.snapshot_source_admitted == Some(false) + && (sequence == 1 && record.status == "pending" + || sequence == 2 && record.status == "unknown") + && record.phase == "launching" => + { + let data: String = db.query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=1 AND type=?2", + [activity, activity::CHANGED], + |row| row.get(0), + )?; + let launched: activity::Record = + serde_json::from_str(&data).map_err(|_| invalid("Invalid preparation origin"))?; + validate_pre_effect(&launched)?; + if origin.version != 1 + || launched.snapshot_origin.as_ref() != Some(origin) + || launched.status != "pending" + || launched.phase != "launching" + || launched.session_id != session + || launched.snapshot_lifetime != record.snapshot_lifetime + || launched.snapshot_request != record.snapshot_request + || launched.snapshot_source_admitted != Some(false) + || origin.activity_id != activity + || origin.session_id != session + || origin.session_created_id != binding.owner.session_created_id + || origin.worktree_id != binding.owner.worktree_id + || origin.binding.directory != binding.owner.directory + || origin.binding.root_identity != binding.root_identity + { + return Err(invalid("Preparation origin differs from its reservation")); + } + } + _ => return Err(invalid("Source effects may already have been admitted")), + } let created: String = db.query_row( "SELECT id FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq LIMIT 1", [session, super::events::CREATED], @@ -136,10 +202,16 @@ fn target(db: &Connection, session: &str, activity: &str) -> Result Result<(), StoreError> { + validate_pre_effect(record)?; + if record.snapshot_origin.is_some() || record.phase != "reserved" { + return Err(invalid("Reservation is not unstarted")); + } + Ok(()) +} +fn validate_pre_effect(record: &activity::Record) -> Result<(), StoreError> { if !record.snapshot_worker || record.binary_document.is_some() - || record.phase != "reserved" - || record.snapshot_origin.is_some() + || record.snapshot_source_admitted == Some(true) || record.snapshot_receipt.is_some() || record.snapshot_restore_receipt.is_some() || record.snapshot_preparation_receipt.is_some() @@ -150,6 +222,61 @@ fn validate_unstarted(record: &activity::Record) -> Result<(), StoreError> { } Ok(()) } +fn acknowledged_target( + db: &Connection, + receipt: &SnapshotPreparationReceipt, +) -> Result { + validate_receipt(receipt)?; + let owner = &receipt.lifetime.owner; + let created: String = db.query_row( + "SELECT id FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq LIMIT 1", + [&owner.session_id, super::events::CREATED], + |row| row.get(0), + )?; + let (request_seq, request_kind, data): (i64, String, String) = db.query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [request_owner(&created, &receipt.request_id)], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + let request: SnapshotPreparationReceipt = + serde_json::from_str(&data).map_err(|_| invalid("Invalid preparation request receipt"))?; + let (sequence, kind, raw): (i64, String, String) = db.query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&receipt.activity_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + let record: activity::Record = + serde_json::from_str(&raw).map_err(|_| invalid("Invalid preparation owner"))?; + if created != owner.session_created_id + || request_seq != 0 + || request_kind != CANCELLED + || request != *receipt + || sequence != receipt.native_sequence + 1 + || kind != activity::CHANGED + || !record.snapshot_worker + || record.binary_document.is_some() + || record.session_id != owner.session_id + || record.status != "settled" + || record.phase != PHASE + || record.snapshot_preparation_receipt.as_ref() != Some(receipt) + || record.snapshot_origin != receipt.origin + || record.snapshot_source_admitted != receipt.source_admitted + || record.snapshot_lifetime.as_ref() != Some(&receipt.lifetime) + || record.snapshot_receipt.is_some() + || record.snapshot_restore_receipt.is_some() + { + return Err(invalid( + "Preparation receipt lacks its exact durable native pair", + )); + } + Ok(Target { + sequence, + raw, + record, + binding: receipt.lifetime.clone(), + }) +} + fn request_owner(created: &str, request: &str) -> String { use sha2::{Digest, Sha256}; let bytes = serde_json::to_vec(&("snapshot-preparation-request-v1", created, request)) @@ -208,9 +335,14 @@ impl Worker { if self.control.stop.is_cancelled() { return Err(invalid("Snapshot preparation cancellation was revoked")); } - self.proof.verify().map_err(|error| invalid(&error))?; + self.proof + .verify_preparation(self.target.record.snapshot_origin.as_ref()) + .map_err(|error| invalid(&error))?; let owner = &self.target.binding.owner; - let current = target(db, &owner.session_id, &owner.activity_id)?; + let current = match self.target.record.snapshot_preparation_receipt.as_ref() { + Some(receipt) => acknowledged_target(db, receipt)?, + None => target(db, &owner.session_id, &owner.activity_id)?, + }; let sequence: i64 = db.query_row( "SELECT MAX(seq) FROM event WHERE aggregate_id=?1", [&owner.session_id], @@ -243,6 +375,8 @@ impl Worker { lifetime: self.target.binding.clone(), native_sequence: self.target.sequence, session_sequence: self.sequence, + origin: self.target.record.snapshot_origin.clone(), + source_admitted: self.target.record.snapshot_source_admitted, }; let mut record = self.target.record.clone(); record.status = "settled".into(); @@ -278,9 +412,11 @@ impl Worker { Ok((events, (self, receipt))) }) .map_err(preparation_error)?; + let mut worker = worker; worker .handle .acknowledge_snapshot_uncertainty(&receipt.activity_id); + worker.proof.acknowledge_preparation(&receipt).map_err(|error| RuntimeError::Conflict(format!("Preparation cancellation committed; native journal acknowledgement requires reconciliation: {error}")))?; Ok::<_, RuntimeError>(receipt) }); let result = joined.await.map_err(|_| { @@ -301,7 +437,7 @@ impl Runtime { self.session_info(session).await?; let session = session.to_owned(); self.inner.store.read(move |db| { - let mut query = db.prepare("SELECT aggregate_id,data FROM event e WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.status')<>'settled' AND json_extract(data,'$.snapshot_origin') IS NULL AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) ORDER BY aggregate_id LIMIT 4097")?; + let mut query = db.prepare("SELECT aggregate_id,data FROM event e WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.status')<>'settled' AND COALESCE(json_extract(data,'$.snapshot_source_admitted'),0)=0 AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) ORDER BY aggregate_id LIMIT 4097")?; let rows = query.query_map([session], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))?; let mut result = Vec::new(); for row in rows { @@ -317,6 +453,7 @@ impl Runtime { &self, session: &str, activity: &str, + acknowledged: Option, ) -> Result { let lifecycle = self.memory_mutation_lease().await?; let handle = self.inner.handle(session).await?; @@ -341,7 +478,10 @@ impl Runtime { .store .read(move |db| { let tx = db.unchecked_transaction()?; - let result = target(&tx, &source, &owner)?; + let result = match acknowledged.as_ref() { + Some(receipt) => acknowledged_target(&tx, receipt)?, + None => target(&tx, &source, &owner)?, + }; tx.commit()?; Ok(result) }) @@ -412,13 +552,14 @@ impl Runtime { session: &str, activity: &str, ) -> Result { - let worker = self.preparation_worker(session, activity).await?; + let worker = self.preparation_worker(session, activity, None).await?; let store = self.inner.store.clone(); tokio::task::spawn_blocking(move || { let worker = store.read(move |db| { let tx = db.unchecked_transaction()?; worker.authorize(&tx)?; tx.commit()?; Ok(worker) }).map_err(preparation_error)?; Ok(SnapshotPreparationReview { activity_id: worker.target.binding.owner.activity_id.clone(), lifetime: worker.target.binding.clone(), native_sequence: worker.target.sequence, session_sequence: worker.sequence, confirmation: worker.fingerprint()?, - warning: "Cancels only this unstarted snapshot preparation. Files and transcript are preserved; checkout and process uncertainty remain independent.".into() }) + origin: worker.target.record.snapshot_origin.clone(), + warning: "Cancels only this pre-effect snapshot preparation. Files and transcript are preserved; checkout and process uncertainty remain independent.".into() }) }).await.map_err(|_| RuntimeError::Conflict("Snapshot preparation review interrupted".into()))? } pub async fn snapshot_preparation_confirm( @@ -449,11 +590,56 @@ impl Runtime { } return Ok(receipt); } - self.preparation_worker(session, activity) + self.preparation_worker(session, activity, None) .await? .confirm(confirmation.clone()) .await } + pub async fn snapshot_preparation_acknowledge( + &self, + session: &str, + activity: &str, + request: &str, + ) -> Result { + let receipt = self + .snapshot_preparation_lookup(session, activity, request) + .await? + .ok_or_else(|| { + RuntimeError::Conflict("No committed preparation cancellation receipt".into()) + })?; + if receipt.origin.is_none() { + return Ok(receipt); + } + let worker = self + .preparation_worker(session, activity, Some(receipt.clone())) + .await?; + let cancelled = worker.control.stop.clone().drop_guard(); + let store = self.inner.store.clone(); + let result = tokio::task::spawn_blocking(move || { + let mut worker = store + .read(move |db| { + let tx = db.unchecked_transaction()?; + worker.authorize(&tx)?; + tx.commit()?; + Ok(worker) + }) + .map_err(preparation_error)?; + worker + .proof + .acknowledge_preparation(&receipt) + .map_err(RuntimeError::Conflict)?; + Ok(receipt) + }) + .await + .map_err(|_| { + RuntimeError::Conflict( + "Preparation journal acknowledgement interrupted; reconcile its retained receipt" + .into(), + ) + })?; + cancelled.disarm(); + result + } pub async fn snapshot_preparation_lookup( &self, session: &str, @@ -493,27 +679,7 @@ impl Runtime { { return Err(invalid("Preparation request belongs to another owner")); } - let (seq, kind, data): (i64, String, String) = tx.query_row( - "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", - [&activity], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - )?; - let native: activity::Record = - serde_json::from_str(&data).map_err(|_| invalid("Invalid preparation owner"))?; - if seq != receipt.native_sequence + 1 - || kind != activity::CHANGED - || !native.snapshot_worker - || native.binary_document.is_some() - || native.session_id != source - || receipt.lifetime.owner.activity_id != activity - || native.status != "settled" - || native.phase != PHASE - || native.snapshot_preparation_receipt.as_ref() != Some(&receipt) - || native.snapshot_origin.is_some() - || native.snapshot_lifetime.as_ref() != Some(&receipt.lifetime) - { - return Err(invalid("Preparation receipt lacks its exact native pair")); - } + acknowledged_target(&tx, &receipt)?; tx.commit()?; Ok(Some(receipt)) }).map_err(preparation_error)?; diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index c12ba7745..c177cd843 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -191,7 +191,11 @@ impl Runtime { "Preparation cancellation lacks its durable pair".into(), )); } - super::SnapshotRecoveryState::CancelledBeforeOrigin + if cancelled.origin.is_some() { + super::SnapshotRecoveryState::CancelledBeforeEffects + } else { + super::SnapshotRecoveryState::CancelledBeforeOrigin + } } ("unknown", None) | ("settled", None) => super::SnapshotRecoveryState::Unknown, ("settled", Some(receipt)) if receipt.request_id == request_id => { diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_record.rs b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs index 03379c834..96b302600 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_record.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_record.rs @@ -110,6 +110,8 @@ pub(super) fn project_pending( .as_ref() .ok_or("Missing resolution origin")?; if reserved.snapshot_request != record.snapshot_request + || reserved.snapshot_lifetime != record.snapshot_lifetime + || reserved.snapshot_source_admitted != record.snapshot_source_admitted || reserved.status != "pending" || origin.activity_id != event.aggregate_id || origin.session_id != record.session_id @@ -125,6 +127,28 @@ pub(super) fn project_pending( return Err("Resolution launch differs from its exact request reservation".into()); } } + 2 if record.phase == "snapshot_source_admitted" + && record.snapshot_source_admitted == Some(true) => + { + let data: String = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=1 AND type=?2", + [&event.aggregate_id, activity::CHANGED], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + let launched: activity::Record = + serde_json::from_str(&data).map_err(|error| error.to_string())?; + if launched.status != "pending" + || launched.phase != "launching" + || launched.snapshot_source_admitted != Some(false) + || launched.snapshot_origin != record.snapshot_origin + || launched.snapshot_lifetime != record.snapshot_lifetime + || launched.snapshot_request != record.snapshot_request + { + return Err("Snapshot source admission differs from its exact launch".into()); + } + } _ => return Err("Invalid snapshot recovery launch sequence".into()), } Ok(true) @@ -178,5 +202,69 @@ pub(super) fn attest( "Native activity differs from the exact snapshot origin".into(), )); } + source_admission(db, origin, &launched, &latest)?; Ok((seq, latest)) } + +fn source_admission( + db: &Connection, + origin: &SnapshotOrigin, + launched: &activity::Record, + latest: &activity::Record, +) -> Result<(), StoreError> { + let invalid = || StoreError::InvalidAppend("Snapshot source-admission evidence changed".into()); + match ( + launched.snapshot_source_admitted, + latest.snapshot_source_admitted, + ) { + (None, None) => return Ok(()), + (Some(false), Some(false) | Some(true)) => {} + _ => return Err(invalid()), + } + let data: String = db.query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=0 AND type=?2", + [&origin.activity_id, activity::CHANGED], + |row| row.get(0), + )?; + let reserved: activity::Record = serde_json::from_str(&data).map_err(|_| invalid())?; + if reserved.snapshot_source_admitted != Some(false) + || !reserved.snapshot_worker + || reserved.binary_document.is_some() + || reserved.status != "pending" + || reserved.phase != "reserved" + || reserved.session_id != origin.session_id + || reserved.snapshot_request != launched.snapshot_request + || reserved.snapshot_lifetime.is_none() + || reserved + .snapshot_lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.version != 2) + || reserved.snapshot_lifetime != launched.snapshot_lifetime + || reserved.snapshot_origin.is_some() + { + return Err(invalid()); + } + if latest.snapshot_source_admitted == Some(false) { + let admitted: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM event WHERE aggregate_id=?1 AND json_extract(data,'$.snapshot_source_admitted')=1)", [&origin.activity_id], |row| row.get(0))?; + if admitted { + return Err(invalid()); + } + } else { + let data: String = db.query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=2 AND type=?2", + [&origin.activity_id, activity::CHANGED], + |row| row.get(0), + )?; + let admitted: activity::Record = serde_json::from_str(&data).map_err(|_| invalid())?; + if admitted.phase != "snapshot_source_admitted" + || admitted.status != "pending" + || admitted.snapshot_source_admitted != Some(true) + || admitted.snapshot_origin.as_ref() != Some(origin) + || admitted.snapshot_lifetime != launched.snapshot_lifetime + || admitted.snapshot_request != launched.snapshot_request + { + return Err(invalid()); + } + } + Ok(()) +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs index b479a21e5..286ddaec1 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs @@ -97,6 +97,7 @@ pub enum SnapshotRecoveryState { Acknowledged, ResolvedByAnotherRequest, CancelledBeforeOrigin, + CancelledBeforeEffects, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 399113202..19d79d720 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -121,6 +121,28 @@ pub struct SnapshotCompletion { } impl SnapshotWork<'_> { + pub fn preparation_record_budget(&self, origin: &SnapshotOrigin) -> Result { + let Some(lifetime) = self.owner.scope.snapshot_lifetime() else { + return Ok(0); + }; + let receipt = super::SnapshotPreparationReceipt { + activity_id: lifetime.owner.activity_id.clone(), + request_id: "\u{1}".repeat(256), + confirmation: "0".repeat(64), + lifetime: lifetime.clone(), + native_sequence: 2, + session_sequence: i64::MAX, + origin: Some(origin.clone()), + source_admitted: Some(false), + }; + let bytes = serde_json::to_vec(&receipt) + .map_err(|error| error.to_string())? + .len() as u64; + if bytes > super::SnapshotPreparationReceipt::MAX_ENCODED_BYTES { + return Err("Preparation cancellation record exceeds limit".into()); + } + Ok(bytes) + } pub fn origin(&self) -> Option<&SnapshotOrigin> { self.owner.scope.snapshot_origin() } @@ -171,6 +193,15 @@ impl SnapshotWork<'_> { } let state = self.owner.admission.0.state.blocking_lock(); self.authorize()?; + self.owner + .scope + .admit_snapshot_source() + .map_err(|error| error.to_string())?; + self.authorize()?; + if let Some(lifetime) = &self.owner._lifetime { + lifetime.admit_source(self.origin().ok_or("Source admission lacks origin")?)?; + } + self.authorize()?; if state.last_seq != self.owner.sequence { return Err("Snapshot Session projection changed before its filesystem effect".into()); } diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 3bd2468bf..022b3b6a5 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -99,12 +99,24 @@ struct LifetimePermit { _permit: tokio::sync::OwnedSemaphorePermit, } impl super::super::SnapshotLifetime for LifetimePermit { + fn admit_source(&self, _: &SnapshotOrigin) -> Result<(), String> { + Ok(()) + } fn binding(&self) -> &super::super::SnapshotLifetimeBinding { &self.binding } fn verify(&self) -> Result<(), String> { Ok(()) } + fn verify_preparation(&self, _: Option<&SnapshotOrigin>) -> Result<(), String> { + Ok(()) + } + fn acknowledge_preparation( + &mut self, + _: &super::super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + Ok(()) + } } impl Snapshots for Lifetimes { fn lock_worker( @@ -143,12 +155,13 @@ impl Snapshots for Lifetimes { } Ok(Some(Box::new(LifetimePermit { binding: super::super::SnapshotLifetimeBinding { - version: 1, + version: 2, owner: request, - root_identity: "fixture-root".into(), + root_identity: "instrumented_root".into(), namespace_identity: "fixture-namespace".into(), directory_identity: "fixture-directory".into(), file_identity: "fixture-file".into(), + source_phase: Some("phase_fixture".into()), }, _permit: permit, }) @@ -206,6 +219,191 @@ fn crash_unstarted(owner: SnapshotWorker) -> String { id } +#[tokio::test] +async fn published_origin_with_positive_false_admission_can_cancel_but_admitted_source_cannot() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + }), + ); + let native_binding = binding(temp.path().canonicalize().unwrap()); + let result = worker(&runtime, &session) + .await + .run(move |work| { + work.begin(native_binding).unwrap(); + Err(RestoreError::Failed( + "cut before intent or source admission".into(), + )) + }) + .await; + assert!(result.is_err()); + let origin: SnapshotOrigin = + serde_json::from_value(latest(&runtime, &session)["snapshot_origin"].clone()).unwrap(); + let review = runtime + .snapshot_preparation_review(&session, &origin.activity_id) + .await + .unwrap(); + assert_eq!(review.origin, Some(origin.clone())); + let receipt = runtime + .snapshot_preparation_confirm( + &session, + &origin.activity_id, + &super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "cancel-before-source".into(), + }, + ) + .await + .unwrap(); + assert_eq!(receipt.source_admitted, Some(false)); + assert_eq!(receipt.origin, Some(origin)); + let admitted = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + assert!( + runtime + .snapshot_preparation_review(&session, &admitted.activity_id) + .await + .is_err() + ); + assert_eq!(latest(&runtime, &session)["snapshot_source_admitted"], true); + runtime.shutdown().await; + base.shutdown().await; +} + +#[tokio::test] +async fn rejected_source_admission_prevents_the_callback_and_retains_positive_preparation() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + }), + ); + let owner = worker(&runtime, &session).await; + let native_binding = binding(temp.path().canonicalize().unwrap()); + let count = Arc::new(AtomicUsize::new(0)); + let calls = count.clone(); + runtime + .inner + .store + .set_append_fault(Some(Arc::new(|_, event| { + event.data["phase"] == "snapshot_source_admitted" + }))); + assert!( + owner + .run(move |work| { + work.begin(native_binding).unwrap(); + assert!( + work.effect(|| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .is_err() + ); + Err(RestoreError::Failed("admission rejected".into())) + }) + .await + .is_err() + ); + runtime.inner.store.set_append_fault(None); + assert_eq!(count.load(Ordering::SeqCst), 0); + let record = latest(&runtime, &session); + assert_eq!(record["snapshot_source_admitted"], false); + let activity = record["snapshot_origin"]["activity_id"].as_str().unwrap(); + runtime + .snapshot_preparation_review(&session, activity) + .await + .unwrap(); + runtime.shutdown().await; + base.shutdown().await; +} + +#[tokio::test] +async fn lost_source_admission_reply_never_calls_the_source_and_never_becomes_false_proof() { + fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("lost source-admission reply".into()), + )) + } + let (temp, base, _, _, _) = super::super::binary_documents_tests::fixture().await; + let store = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(temp.path().join("source-admission.db")), + Runtime::registry(), + )) + .unwrap(); + let runtime = runtime_with_snapshots( + &base, + Arc::new(store), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + }), + ); + let session = runtime + .create_session(super::super::CreateSession { + directory: temp.path().to_str().unwrap().into(), + model: "test/main".into(), + ..Default::default() + }) + .await + .unwrap() + .id; + let owner = worker(&runtime, &session).await; + let id = owner.scope.id().to_owned(); + let binding = binding(temp.path().canonicalize().unwrap()); + let store = runtime.inner.store.clone(); + let calls = Arc::new(AtomicUsize::new(0)); + let effects = calls.clone(); + assert!( + owner + .run(move |work| { + work.begin(binding).unwrap(); + store + .transaction(|tx| { + tx.wal_hook(Some(lost_reply)); + Ok(()) + }) + .unwrap(); + assert!( + work.effect(|| { + effects.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .is_err() + ); + Err(RestoreError::Failed( + "admission outcome requires journal recovery".into(), + )) + }) + .await + .is_err() + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + let record = latest(&runtime, &session); + assert_eq!(record["snapshot_source_admitted"], true); + assert_eq!(record["phase"], "snapshot_source_admitted"); + assert!( + runtime + .snapshot_preparation_review(&session, &id) + .await + .is_err() + ); + runtime.shutdown().await; + base.shutdown().await; +} + #[tokio::test] async fn preparation_lost_commit_reply_is_lookup_only_after_reopening_the_real_store() { fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { @@ -932,6 +1130,7 @@ async fn failed_snapshot(runtime: &Runtime, session: &str, directory: PathBuf) - owner .run(move |work| { work.begin(binding(directory)).unwrap(); + let _ = work.effect(|| Err::<(), String>("interrupted source operation".into())); Err(RestoreError::Failed("interrupted native restore".into())) }) .await diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index daff83c07..a6bcdf2ec 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -9,6 +9,14 @@ use std::{ const RECORD_LIMIT: u64 = 16 * 1024; +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct SourcePhase { + lifetime: SnapshotLifetimeBinding, + admitted: bool, + origin: Option, +} + pub(crate) struct Lifetime { binding: SnapshotLifetimeBinding, file: File, @@ -38,6 +46,64 @@ impl SnapshotLifetime for Lifetime { .ok_or("Snapshot lifetime namespace is missing")?; store.verify_lifetime(&self.binding) } + fn verify_preparation( + &self, + origin: Option<&cyber_server::runtime::SnapshotOrigin>, + ) -> Result<(), String> { + self.verify()?; + let store = Store::existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Missing preparation namespace")?; + store.verify_native_preparation(&self.binding)?; + if let Some(origin) = origin { + if origin.activity_id != self.binding.owner.activity_id + || origin.session_id != self.binding.owner.session_id + || origin.session_created_id != self.binding.owner.session_created_id + || origin.worktree_id != self.binding.owner.worktree_id + { + return Err("Foreign preparation origin".into()); + } + let store = Store::existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Missing preparation namespace")?; + store.verify_preparation_origin(origin)?; + } + Ok(()) + } + fn admit_source(&self, origin: &cyber_server::runtime::SnapshotOrigin) -> Result<(), String> { + self.verify()?; + let store = Store::existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Missing source-admission namespace")?; + store.record_source_admission(&self.binding, origin) + } + fn acknowledge_preparation( + &mut self, + receipt: &cyber_server::runtime::SnapshotPreparationReceipt, + ) -> Result<(), String> { + if receipt.lifetime != self.binding { + return Err("Foreign preparation receipt".into()); + } + self.verify_preparation(receipt.origin.as_ref())?; + if receipt.origin.is_some() { + let store = Store::recover_existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )?; + store.acknowledge_preparation(receipt)?; + } + Ok(()) + } } fn lock(file: &File) -> Result<(), String> { @@ -88,18 +154,30 @@ impl Store { if binding.owner != owner { return Err("Snapshot lifetime owner changed".into()); } + self.verify_native_preparation(&binding)?; return self.lock_lifetime(&binding); } let mut binding = SnapshotLifetimeBinding { - version: 1, + version: 2, owner, root_identity: self.root_id.clone(), namespace_identity: storage::dir_identity(&self.namespace)?, directory_identity: storage::dir_identity(&directory)?, file_identity: "0".repeat(storage::IDENTITY_MAX_BYTES), + source_phase: Some(format!("phase_{}", ulid::Ulid::new())), }; - let budget = serde_json::to_vec(&binding).map_err(storage::fail)?.len() as u64; - if budget > RECORD_LIMIT { + let phase_budget = serde_json::to_vec(&SourcePhase { + lifetime: binding.clone(), + admitted: false, + origin: None, + }) + .map_err(storage::fail)? + .len() as u64; + let binding_budget = serde_json::to_vec(&binding).map_err(storage::fail)?.len() as u64; + let budget = binding_budget + .checked_add(phase_budget) + .ok_or("Snapshot lifetime size overflow")?; + if binding_budget > RECORD_LIMIT || phase_budget > RECORD_LIMIT { return Err("Snapshot lifetime record exceeds limit".into()); } self.reserve(budget)?; @@ -112,6 +190,20 @@ impl Store { let bytes = serde_json::to_vec(&binding).map_err(storage::fail)?; file.write_all(&bytes).map_err(storage::fail)?; storage::sync_file(&file)?; + let phase = SourcePhase { + lifetime: binding.clone(), + admitted: false, + origin: None, + }; + storage::create( + &directory, + binding + .source_phase + .as_deref() + .ok_or("Missing source phase")?, + &serde_json::to_vec(&phase).map_err(storage::fail)?, + )?; + storage::sync(&directory)?; before_publish()?; rustix::fs::renameat_with( &directory, @@ -155,7 +247,7 @@ impl Store { fn verify_lifetime(&self, binding: &SnapshotLifetimeBinding) -> Result<(), String> { valid_owner(&binding.owner)?; self.verify_root()?; - if binding.version != 1 + if !matches!(binding.version, 1 | 2) || binding.owner.directory != self.root || binding.root_identity != self.root_id || binding.namespace_identity != storage::dir_identity(&self.namespace)? @@ -184,6 +276,98 @@ impl Store { } Ok(()) } + + fn read_source_phase(&self, binding: &SnapshotLifetimeBinding) -> Result { + self.verify_lifetime(binding)?; + let name = binding + .source_phase + .as_deref() + .ok_or("Legacy worker has no native source phase")?; + if binding.version != 2 + || !name.starts_with("phase_") + || name.len() > 128 + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err("Unsupported native source phase".into()); + } + let directory = storage::existing_child(&self.namespace, "workers", true)?; + let file = storage::open(&directory, Path::new(name))?; + storage::validate_lock(&file)?; + let mut bytes = Vec::new(); + file.take(cyber_server::runtime::SnapshotPreparationReceipt::MAX_ENCODED_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(storage::fail)?; + if bytes.len() as u64 > cyber_server::runtime::SnapshotPreparationReceipt::MAX_ENCODED_BYTES + { + return Err("Native source phase exceeds limit".into()); + } + let phase: SourcePhase = serde_json::from_slice(&bytes).map_err(storage::fail)?; + if phase.lifetime != *binding || phase.admitted != phase.origin.is_some() { + return Err("Native source phase changed owner".into()); + } + Ok(phase) + } + pub(super) fn verify_native_preparation( + &self, + binding: &SnapshotLifetimeBinding, + ) -> Result<(), String> { + if self.read_source_phase(binding)?.admitted { + return Err("Native source effects may already have been admitted".into()); + } + Ok(()) + } + fn record_source_admission( + &self, + binding: &SnapshotLifetimeBinding, + origin: &cyber_server::runtime::SnapshotOrigin, + ) -> Result<(), String> { + let phase = self.read_source_phase(binding)?; + if origin.activity_id != binding.owner.activity_id + || origin.session_id != binding.owner.session_id + || origin.session_created_id != binding.owner.session_created_id + || origin.worktree_id != binding.owner.worktree_id + { + return Err("Foreign native source admission".into()); + } + self.verify_worker_namespace(origin)?; + let (_, journal) = self.journal(&origin.binding.journal_id)?; + if journal.origin.as_ref() != Some(origin) || journal.ownership.is_none() { + return Err("Source admission lacks prepared native intent".into()); + } + if phase.admitted { + return if phase.origin.as_ref() == Some(origin) { + Ok(()) + } else { + Err("Native source origin changed".into()) + }; + } + let phase = SourcePhase { + lifetime: binding.clone(), + admitted: true, + origin: Some(origin.clone()), + }; + let bytes = serde_json::to_vec(&phase).map_err(storage::fail)?; + if bytes.len() as u64 > cyber_server::runtime::SnapshotPreparationReceipt::MAX_ENCODED_BYTES + { + return Err("Native source phase exceeds limit".into()); + } + self.reserve( + (bytes.len() as u64) + .checked_mul(2) + .ok_or("Snapshot source size overflow")?, + )?; + let directory = storage::existing_child(&self.namespace, "workers", true)?; + storage::atomic( + &directory, + binding + .source_phase + .as_deref() + .ok_or("Missing source phase")?, + &bytes, + ) + } } #[cfg(test)] @@ -258,6 +442,39 @@ mod tests { assert!(!file.exists()); } + #[test] + fn legacy_lifetime_stays_inspectable_but_never_grants_preparation_cancellation() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let lifetime = store.reserve_lifetime(request(&store.root)).unwrap(); + let mut binding = lifetime.binding().clone(); + drop(lifetime); + // Reproduce a genuine version-1 record in the same private inode. It + // has valid lifetime ownership, but predates native source-phase proof. + binding.version = 1; + binding.source_phase = None; + let workers = storage::existing_child(&store.namespace, "workers", true).unwrap(); + let mut file = workers + .open_with( + &binding.owner.activity_id, + cap_std::fs::OpenOptions::new().write(true).truncate(true), + ) + .unwrap(); + file.write_all(&serde_json::to_vec(&binding).unwrap()) + .unwrap(); + file.sync_all().unwrap(); + let legacy = store.lock_lifetime(&binding).unwrap(); + legacy.verify().unwrap(); + assert!(legacy.verify_preparation(None).is_err()); + drop(legacy); + assert!(store.reserve_lifetime(binding.owner.clone()).is_err()); + let retained: SnapshotLifetimeBinding = serde_json::from_slice( + &storage::read(&workers, &binding.owner.activity_id, RECORD_LIMIT).unwrap(), + ) + .unwrap(); + assert_eq!(retained, binding); + } + #[test] fn lifetime_cap_refuses_before_creating_an_owner_file_and_gc_retains_preparation() { let temp = tempfile::tempdir().unwrap(); @@ -333,15 +550,26 @@ mod tests { let store = store(temp.path()); let directory = storage::existing_child(&store.namespace, "workers", true).unwrap(); let stages = storage::names(&directory, 10).unwrap(); - assert_eq!(stages.len(), 1); - assert!(stages[0].starts_with("preparing_")); - let bytes = storage::read(&directory, &stages[0], RECORD_LIMIT).unwrap(); + assert_eq!(stages.len(), 2); + assert!(stages.iter().any(|name| name.starts_with("preparing_"))); + assert!(stages.iter().any(|name| name.starts_with("phase_"))); + let retained: Vec<_> = stages + .iter() + .map(|name| { + ( + name.clone(), + storage::read(&directory, name, RECORD_LIMIT).unwrap(), + ) + }) + .collect(); let lifetime = store.reserve_lifetime(request(&store.root)).unwrap(); - assert_eq!( - storage::read(&directory, &stages[0], RECORD_LIMIT).unwrap(), - bytes - ); - assert_eq!(storage::names(&directory, 10).unwrap().len(), 2); + for (name, bytes) in retained { + assert_eq!( + storage::read(&directory, &name, RECORD_LIMIT).unwrap(), + bytes + ); + } + assert_eq!(storage::names(&directory, 10).unwrap().len(), 4); lifetime.verify().unwrap(); } diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index b0dfbec6d..3fdf61666 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,5 +1,6 @@ //! Private BLAKE3 snapshots for exact non-Git Locations; no user repository required. mod ownership; +mod preparation; mod resolution; mod restore; mod storage; diff --git a/crates/cyber-snapshot/src/copy/preparation.rs b/crates/cyber-snapshot/src/copy/preparation.rs new file mode 100644 index 000000000..e12e9f775 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/preparation.rs @@ -0,0 +1,269 @@ +//! Metadata settlement of a positive runtime pre-effect cancellation. +use super::{Store, storage}; +use cap_fs_ext::DirExt; +use cyber_server::runtime::{SnapshotOrigin, SnapshotPreparationReceipt}; +pub(super) const FILE: &str = "preparation.json"; +pub(super) const LIMIT: u64 = SnapshotPreparationReceipt::MAX_ENCODED_BYTES; + +impl Store { + pub(super) fn verify_preparation_origin(&self, origin: &SnapshotOrigin) -> Result<(), String> { + self.verify_worker_namespace(origin)?; + let directory = self + .journals + .open_dir_nofollow(&origin.binding.journal_id) + .map_err(storage::fail)?; + match directory.symlink_metadata("intent.json") { + Ok(_) => { + let (_, journal) = self.journal(&origin.binding.journal_id)?; + if journal.origin.as_ref() != Some(origin) + || !journal.applied.is_empty() + || journal.acknowledged + || journal.settled_by.is_some() + || journal.completed && !journal.changes.is_empty() + { + return Err("Intent contradicts positive pre-effect cancellation".into()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(storage::fail(error)), + } + Ok(()) + } + pub(crate) fn acknowledge_preparation( + &self, + receipt: &SnapshotPreparationReceipt, + ) -> Result<(), String> { + self.verify_native_preparation(&receipt.lifetime)?; + let origin = receipt + .origin + .as_ref() + .ok_or("Cancellation has no journal origin")?; + if receipt.source_admitted != Some(false) + || origin.activity_id != receipt.activity_id + || origin.session_id != receipt.lifetime.owner.session_id + || origin.session_created_id != receipt.lifetime.owner.session_created_id + || origin.binding.root_identity != receipt.lifetime.root_identity + { + return Err("Cancellation has no positive pre-effect proof".into()); + } + self.verify_preparation_origin(origin)?; + if let Some(existing) = self.preparation_cancellation(&origin.binding.journal_id)? { + return if existing == *receipt { + Ok(()) + } else { + Err("Journal has another cancellation receipt".into()) + }; + } + let bytes = serde_json::to_vec(receipt).map_err(storage::fail)?; + if bytes.len() as u64 > LIMIT { + return Err("Preparation cancellation record exceeds limit".into()); + } + self.reserve( + (bytes.len() as u64) + .checked_mul(2) + .ok_or("Snapshot size overflow")?, + )?; + let directory = self + .journals + .open_dir_nofollow(&origin.binding.journal_id) + .map_err(storage::fail)?; + self.verify_preparation_origin(origin)?; + storage::atomic(&directory, FILE, &bytes) + } + pub(super) fn preparation_cancellation( + &self, + id: &str, + ) -> Result, String> { + let directory = self.journals.open_dir_nofollow(id).map_err(storage::fail)?; + match directory.symlink_metadata(FILE) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(storage::fail(error)), + } + let receipt: SnapshotPreparationReceipt = + serde_json::from_slice(&storage::read(&directory, FILE, LIMIT)?) + .map_err(storage::fail)?; + self.verify_native_preparation(&receipt.lifetime)?; + let origin = receipt + .origin + .as_ref() + .ok_or("Cancellation lacks original journal binding")?; + if receipt.source_admitted != Some(false) + || origin.binding.journal_id != id + || origin.activity_id != receipt.activity_id + || receipt.lifetime.owner.activity_id != receipt.activity_id + || origin.session_id != receipt.lifetime.owner.session_id + || origin.session_created_id != receipt.lifetime.owner.session_created_id + { + return Err("Invalid preparation cancellation binding".into()); + } + self.verify_preparation_origin(origin)?; + Ok(Some(receipt)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use cyber_server::runtime::{SnapshotBinding, SnapshotLifetime, SnapshotLifetimeRequest}; + fn setup() -> ( + tempfile::TempDir, + Store, + super::super::lifetime::Lifetime, + SnapshotPreparationReceipt, + ) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work"); + std::fs::create_dir(&root).unwrap(); + let store = Store::open( + &temp.path().join("data"), + root.to_str().unwrap(), + super::super::Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + let lifetime = store + .reserve_lifetime(SnapshotLifetimeRequest { + activity_id: "op_preparation".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: store.root.clone(), + worktree_id: None, + }) + .unwrap(); + let dir = storage::private_child(&store.journals, "restore_preparation").unwrap(); + let origin = SnapshotOrigin { + version: 1, + activity_id: "op_preparation".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + worktree_id: None, + binding: SnapshotBinding { + directory: store.root.clone(), + root_identity: store.root_id.clone(), + namespace_identity: storage::dir_identity(&store.journals).unwrap(), + journal_id: "restore_preparation".into(), + journal_identity: storage::dir_identity(&dir).unwrap(), + lock_identity: Some(store.lock_identity().unwrap()), + }, + }; + let receipt = SnapshotPreparationReceipt { + activity_id: origin.activity_id.clone(), + request_id: "cancel-preparation".into(), + confirmation: "a".repeat(64), + lifetime: lifetime.binding().clone(), + native_sequence: 1, + session_sequence: 8, + origin: Some(origin), + source_admitted: Some(false), + }; + (temp, store, lifetime, receipt) + } + #[test] + fn origin_without_intent_is_acknowledged_without_source_scan_or_fabricated_completion() { + let (temp, store, mut lifetime, receipt) = setup(); + let target = temp.path().join("owner-target"); + std::fs::write(&target, b"later owner bytes").unwrap(); + std::os::unix::fs::symlink(&target, store.root.join("source")).unwrap(); + assert!(store.ensure_recovered().is_err()); + drop(store); + lifetime.acknowledge_preparation(&receipt).unwrap(); + lifetime.acknowledge_preparation(&receipt).unwrap(); + let store = Store::recover_existing( + &temp.path().join("data"), + temp.path().join("work").to_str().unwrap(), + super::super::Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + store.ensure_recovered().unwrap(); + assert!(store.recovery().unwrap().is_empty()); + assert!( + store.journal("restore_preparation").is_err(), + "cancellation invented a restore intent" + ); + assert!( + std::fs::symlink_metadata(store.root.join("source")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(std::fs::read(target).unwrap(), b"later owner bytes"); + } + #[test] + fn contradictory_intent_and_admitted_receipt_never_publish_cancellation_metadata() { + let (_temp, store, _lifetime, mut receipt) = setup(); + receipt.source_admitted = Some(true); + assert!(store.acknowledge_preparation(&receipt).is_err()); + receipt.source_admitted = Some(false); + let dir = store + .journals + .open_dir_nofollow("restore_preparation") + .unwrap(); + storage::create(&dir, "intent.json", b"corrupt intent evidence").unwrap(); + assert!(store.acknowledge_preparation(&receipt).is_err()); + assert!(dir.symlink_metadata(FILE).is_err()); + assert_eq!( + storage::read(&dir, "intent.json", 100).unwrap(), + b"corrupt intent evidence" + ); + } + #[test] + fn cancellation_metadata_obeys_lowered_cap_and_retries_without_deleting_evidence() { + let (_temp, mut store, _lifetime, receipt) = setup(); + store.options.max_total = 1; + assert!(store.acknowledge_preparation(&receipt).is_err()); + assert!(store.ensure_recovered().is_err()); + store.options.max_total = 4 * 1024 * 1024; + store.acknowledge_preparation(&receipt).unwrap(); + assert_eq!( + store + .preparation_cancellation("restore_preparation") + .unwrap(), + Some(receipt) + ); + } + #[test] + fn native_admission_survives_an_older_database_phase_and_missing_phase_is_never_false_proof() { + let (_temp, store, lifetime, receipt) = setup(); + let origin = receipt.origin.as_ref().unwrap(); + let dir = store + .journals + .open_dir_nofollow(&origin.binding.journal_id) + .unwrap(); + let mut journal = store + .prepare_journal(vec![], &origin.binding.journal_id, None, None, None) + .unwrap(); + journal.origin = Some(origin.clone()); + journal.ownership = + Some(super::super::ownership::Ownership::prepare(&store, &dir, &[]).unwrap()); + Store::record(&dir, &journal).unwrap(); + lifetime.admit_source(origin).unwrap(); + // A stale/rolled-back database receipt still says false. Native evidence + // independently refuses cancellation and is never reset by another call. + assert!(lifetime.verify_preparation(None).is_err()); + assert!(store.acknowledge_preparation(&receipt).is_err()); + assert!(dir.symlink_metadata(FILE).is_err()); + let workers = storage::existing_child(&store.namespace, "workers", true).unwrap(); + let phase = receipt.lifetime.source_phase.as_ref().unwrap(); + workers.remove_file(phase).unwrap(); + assert!(lifetime.verify_preparation(None).is_err()); + assert!( + store + .reserve_lifetime(receipt.lifetime.owner.clone()) + .is_err() + ); + assert!( + workers.symlink_metadata(phase).is_err(), + "missing source phase was recreated" + ); + } +} diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 5bc95f19c..57cc7d61d 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -58,6 +58,9 @@ pub(super) struct RestoreContext { impl Store { pub(super) fn ensure_recovered(&self) -> Result<()> { for id in storage::names(&self.journals, ENTRY_LIMIT)? { + if self.preparation_cancellation(&id)?.is_some() { + continue; + } let (_, record) = self.journal(&id)?; if record.settled_by.is_some() { self.verify_settlement(&record)?; @@ -146,6 +149,11 @@ impl Store { let mut result = vec![]; let mut response_bytes = 32usize; for id in storage::names(&self.journals, ENTRY_LIMIT)? { + // Cancelled preparations are not completed restores. Their receipt is + // exposed by the preparation owner API; all private evidence remains. + if self.preparation_cancellation(&id)?.is_some() { + continue; + } let (_, record) = self.journal(&id)?; let view = super::Recovery { id, @@ -467,6 +475,13 @@ impl Store { }) .transpose()?; let ownership = Some(Ownership::budget(&changes)?); + let cancellation_reserve = match (worker, origin.as_ref()) { + (Some(worker), Some(origin)) => worker + .preparation_record_budget(origin)? + .checked_mul(2) + .ok_or("Restore size overflow")?, + _ => 0, + }; if let Some(resolution) = &mut resolution { resolution.outcome = Some(resolution.review.budget()); } @@ -517,6 +532,7 @@ impl Store { bytes .checked_mul(2) .and_then(|bytes| bytes.checked_add(parent_reserve)) + .and_then(|bytes| bytes.checked_add(cancellation_reserve)) .ok_or("Restore size overflow")?, |sum, length| sum.checked_add(length).ok_or("Restore size overflow"), )?; @@ -563,6 +579,16 @@ impl Store { pub(super) fn verify_worker_namespace(&self, origin: &SnapshotOrigin) -> Result<()> { self.verify_root()?; let binding = &origin.binding; + if origin.version != 1 + || !binding.journal_id.starts_with("restore_") + || binding.journal_id.len() > 256 + || !binding + .journal_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err("Invalid snapshot worker journal binding".into()); + } self.verify_lock_route( binding .lock_identity diff --git a/crates/cyber-store/migrations/20261106010000_snapshot_source_admission.sql b/crates/cyber-store/migrations/20261106010000_snapshot_source_admission.sql new file mode 100644 index 000000000..3c3b499bd --- /dev/null +++ b/crates/cyber-store/migrations/20261106010000_snapshot_source_admission.sql @@ -0,0 +1,4 @@ +-- Compatibility marker: source-admission phases and pre-effect cancellation +-- receipts must not be interpreted by older snapshot writers. Preserve all +-- existing evidence and use the unknown-migration gate to require an upgrade. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index 88760a756..fb2e16e1f 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -177,6 +177,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261106000000_snapshot_worker_ownership", sql: include_str!("../migrations/20261106000000_snapshot_worker_ownership.sql"), }, + Migration { + id: "20261106010000_snapshot_source_admission", + sql: include_str!("../migrations/20261106010000_snapshot_source_admission.sql"), + }, ]; pub(crate) enum Outcome { @@ -306,7 +310,13 @@ mod tests { let Outcome::TooNew(unknown) = migrate(&mut reopened, legacy).unwrap() else { panic!("older binaries accepted snapshot-worker storage for mutation"); }; - assert_eq!(unknown, ["20261106000000_snapshot_worker_ownership"]); + assert_eq!( + unknown, + MIGRATIONS[at..] + .iter() + .map(|migration| migration.id) + .collect::>() + ); let retained: String = reopened .query_row( "SELECT data FROM event WHERE id='evt_retained'", @@ -317,6 +327,40 @@ mod tests { assert_eq!(retained, evidence); } + #[test] + fn source_admission_marker_preserves_evidence_and_rejects_the_previous_worker_catalog() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("source-admission.db"); + let at = MIGRATIONS + .iter() + .position(|migration| migration.id == "20261106010000_snapshot_source_admission") + .unwrap(); + let legacy = &MIGRATIONS[..at]; + let mut conn = Connection::open(&path).unwrap(); + assert!(matches!( + migrate(&mut conn, legacy).unwrap(), + Outcome::Current + )); + let evidence = r#"{"session_id":"ses_retained","snapshot_worker":true,"status":"unknown","snapshot_source_admitted":true}"#; + conn.execute("INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('evt_source','op_source',0,'native.activity.changed.1',?1,1)", [evidence]).unwrap(); + assert!(matches!( + migrate(&mut conn, MIGRATIONS).unwrap(), + Outcome::Current + )); + drop(conn); + let mut reopened = Connection::open(path).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut reopened, legacy).unwrap() else { + panic!("previous worker catalog admitted source-admission storage"); + }; + assert_eq!(unknown, ["20261106010000_snapshot_source_admission"]); + let retained: String = reopened + .query_row("SELECT data FROM event WHERE id='evt_source'", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(retained, evidence); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 076cf5bb6..1cbf838c7 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -148,3 +148,12 @@ keep/rollback, reconciliation, Git journaling, reclamation and full native/clien acceptance remain required. Approval and current evidence are recorded in the daily coding change's tasks.md and implementation.md; publishing a prerequisite does not complete recovery or either issue. + + +The local pre-effect recovery continuation adds version-2 native source phases, +positive cancellation of an exited published origin before source admission, and +metadata acknowledgement reconciliation. Database and native admission evidence +must both permit cancellation; a database backup cannot erase native admission. +Original intents and payloads remain retained. This is progress on the approved +recovery protocol, with independent checkout ownership, Git journaling, safe cleanup, +complete clients and full platform/cut acceptance still required before delivery. diff --git a/docs/rewind.md b/docs/rewind.md index 89f8c540e..3891f9dfa 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -249,8 +249,8 @@ child. These operations preserve later files, including unsupported replacements and do not move the transcript. Historical receipts without immutable proof cannot authorize new journal writes. -If a crash occurred before an origin or restore journal was published, list the -retained preparations and review the selected activity: +If a crash occurred before source effects were admitted, list the retained +preparation candidates and review the selected activity: ```sh cyber api GET /sessions//rewind/preparations @@ -261,18 +261,29 @@ cyber api GET /sessions//rewind/preparations//requests/ ``` The SDK exposes `snapshotPreparations`, `snapshotPreparationReview`, -`snapshotPreparationConfirm` and `snapshotPreparationLookup` on `client.session`. -Review requires an exact retained lifetime lock proving that this worker exited, -a positive unstarted reservation, and no published origin. Confirmation cancels -only that preparation: source files and transcript remain unchanged. Live workers, -foreign or changed evidence, published origins and legacy reservations without -lifetime proof refuse. Checkout/process uncertainty remains independent. Cancelling -an unstarted keep/rollback request leaves its original failed restore unresolved; -that request's lookup reports `cancelled_before_origin`. Retrying a cancellation -uses its original durable receipt without reopening native proof. - -Created source directories, private payloads and lifetime preparations remain -protected for acknowledged safe cleanup. Do not edit records or remove evidence -to force a retry. Origin-before-intent recovery, independent managed-checkout proof, -Git journaling, reclamation, typed recovery clients and broader native crash/platform +`snapshotPreparationConfirm`, `snapshotPreparationLookup` and +`snapshotPreparationAcknowledge` on `client.session`. Listing does not access native +files; review determines eligibility. Review requires the exact retained lifetime +lock proving that this worker exited and a version-2 native phase record proving +that source effects were never admitted. A published origin is eligible only when +the database also positively records no source admission. Missing journal files +alone grant no authority. An older database backup cannot override an admitted +native phase. Live workers, changed proof, legacy reservations and admitted source +effects refuse cancellation. + +Confirmation cancels only that preparation: source files and transcript remain +unchanged. Its paired durable receipt precedes native cancellation metadata; the +original intent is retained without fabricating a completed restore. Retry and +lookup return the original durable pair without reopening native proof. After a +committed receipt with missing native acknowledgement, POST +`/sessions//rewind/preparations//requests//acknowledge` +to reconcile metadata under the exact proof. This never replays source changes. +Checkout/process uncertainty remains independent. Cancelling an unstarted +keep/rollback request leaves its original failed restore unresolved; its lookup +reports `cancelled_before_origin` or `cancelled_before_effects`. + +Created source directories, private payloads, native phase records and lifetime +preparations remain protected for acknowledged safe cleanup. Do not edit records +or remove evidence to force a retry. Independent managed-checkout proof, Git +journaling, reclamation, typed recovery clients and broader native crash/platform acceptance remain unfinished. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index bcb9fe1c3..0318c12ce 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1094,3 +1094,84 @@ scope is ordinary `append`; it does not intercept checked/paired appends. Writer SQLite triggers now model the abrupt disposal and second-owner insertion cuts, and the final suites above pass. No production Store fault semantics were changed. This increment remains local and does not complete M2–M8, issue #93 or issue #92. + + +### Published-origin cancellation before source admission (2026-11-06, local) + +This continuation extends the exited-preparation path to a published origin with +positive unadmitted evidence. Before its first source callback a new file-copy worker +commits native activity sequence 2 with `snapshot_source_admitted: true`, publishes +the exact origin and admitted phase in private native metadata, then rechecks +current authority. The local scope advances only after observing the Store reply; +a lost reply cannot run the callback or overwrite durable admission with disposal. +All native resources remain inside the actual worker through final filesystem work. + +Review found database-only false evidence insufficient after restoration of an older +database backup. Version-2 lifetime bindings now name a native phase file, flushed +before stable lifetime publication. It can only advance to admitted, never reset. +Cancellation requires an exited exact lifetime lock, positive native false evidence, +and, for an origin, exact database false reservation/launch history. Missing, +legacy or admitted native phase refuses. Historical cancellation receipts remain +readable without acquiring new authority. The compatibility marker prevents a +previous worker-catalog binary from mutating this Store while retaining event bytes. + +The paired Store cancellation retains the original activity, request, creation, +Session revision, origin and lifetime proof. Only after its observed commit does +native metadata receive `preparation.json`; the original intent remains unchanged. +The new preparation request `/acknowledge` route reconciles metadata without source +mutation, inference or transcript movement. Lookup and duplicate confirmation only +read the original durable pair. A cancelled resolution reports +`cancelled_before_effects` when it had an origin; its failed parent remains fenced. +Quota includes native phases, orphan construction files and cancellation receipts; +existing conservative GC protection remains, with reclamation still incomplete. + +Regression coverage includes rejected admission and a real SQLite committed-but-lost +reply before callback dispatch, native database-backup disagreement/missing phase, +legacy lifetime refusal, cap/retry, missing/corrupt intent and later symlink owner +bytes. A real service kill observes the exact origin/false admission boundary over +1280 binary source files; cancellation/restart/retry preserves later owner edits +and transcript, then permits an ordinary prompt in the unmanaged non-Git fixture. +This real-process observation does not prove the finer before-intent cut; the native +missing-intent case is separate. Independent checkout-resource proof, Git-native +recovery, reclamation, complete clients and full M8 platform/cut coverage remain +open. Neither issue nor the whole recovery delivery is complete or merge-ready. + + +Local verification for this continuation: 416 server unit tests passed (one ignored +existing subprocess fixture), plus all 22 affected integrations (4 idle cancellation, +13 Location activity, 5 rewind preview). The complete snapshot suite passed 46 unit, +18 file-copy and 17 Git cases; the added legacy-lifetime regression was then included +in a passing 47-case native unit run, with its subprocess child fixture listed +ignored and exercised by its parent tests. All 16 real CLI branch/recovery tests +passed; the two early-kill cases were rerun successfully after adding explicit +acknowledgement calls before/after restart and original-intent/tombstone assertions. +Store tests passed all 59 cases, including the previous-catalog compatibility gate. +OpenAPI consistency and generated SDK checks passed; SDK typechecking and 134 tests +passed, with 10 configured process tests skipped because `CYBER_BIN` was unset. +Strict pinned OpenSpec passed 117 items; lint reports 0 errors/25 warnings and the +inventory remains 53 capabilities/1030 requirements. Formatting and diff checks +passed. Server/snapshot/CLI all-target Clippy passed with warnings denied, followed +by another passing server/snapshot check after the final port review. Native +sandbox deprecation and Darwin linker warnings were observed without +assigning an origin to main. Linux/Windows, full-workspace and TUI/App recovery +acceptance were not run here. + +The code-review skill was applied to every changed runtime, port, native adapter, +Store migration, HTTP, test, documentation and generated SDK file. The runtime/native +port separation fits the repository. Review checked exact lifetime and journal +routes, private file identities/modes/aliases, positive source-admission history, +writer resource retention and lost replies, request identity/lookup, current owner +authority, encoded/quota bounds, legacy compatibility and no source/transcript replay. +It corrected database-only proof by adding native phases and made the default +adapter preparation verification refuse without an explicit implementation. New +public acknowledgement coverage reaches the actual route through the real CLI; +metadata cap/retry and missing-intent behavior also have native regression tests. +No critical or major findings remain for this continuation; complete recovery still +has the parent acceptance gaps described above. + +Remote refresh confirmed main remains `db9e171e`, issues #92/#93 are open and no +PRs are open. This continuation remains local; no merge or accepted delivery is +claimed. The next ownership gap is the managed checkout acquisition/settlement cut: +its Session-owned claim must become independent exact worker evidence before an +exited preparation can safely settle that resource. No existing unknown lease may +be cleared merely because the snapshot worker exited. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 399032499..020ca2a15 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -12,8 +12,9 @@ Implementation checkpoint: the non-Git review/confirm/lookup API and SDK now use durable request reservation and paired native settlement. Local real-service partial-restore kills exercise keep/rollback and lookup without replay. Explicit journal acknowledgement reconciliation is implemented for new file-copy receipts; -reserved-before-origin cancellation is available for new lifetime-bound file-copy -workers. Origin-before-intent and independent resource cuts, Git journaling, +pre-effect cancellation is available for new version-2 lifetime-bound file-copy +workers, including a published origin with no admitted source effects. Independent +resource cuts, Git journaling, reclamation, complete clients and the platform matrix remain unfinished. Parent tasks are open; the complete recovery delivery is not merge-ready. @@ -238,27 +239,44 @@ must eventually account for both complete records and orphan stages. This supplies lifetime evidence for the reserved-before-origin cut. It does not by itself admit cancellation/acknowledgement of that reservation, prove that a missing journal had no source effects, or acknowledge managed checkout/process -uncertainty. The explicit pre-effect consuming protocol, origin-before-intent cut, -Git equivalent and complete native/client acceptance remain required. Older -reservations without this binding receive no retrospective proof. +uncertainty. The following consuming path adds positive native phase proof and +paired cancellation; independent resource ownership, Git equivalents and complete +native/client acceptance remain required. Older reservations without this binding +receive no retrospective proof. -### Consuming a reservation that never published an origin +### Consuming an exited preparation before source effects -The pre-origin owner path reviews a specific native activity and confirms a -fingerprint before cancelling that preparation. Eligibility requires the positive -sequence-zero snapshot reservation, its unchanged exact lifetime binding, no -historical published origin, and no more than its unlaunched disposal record. +The owner path reviews a specific native activity and confirms a fingerprint +before cancelling its preparation. Eligibility requires the positive sequence-zero +snapshot reservation, unchanged exact version-2 lifetime binding and an unadmitted +native source phase. A published origin additionally requires explicit false +source-admission evidence in both reservation and launching records. Only the +original pending record or its exact unadmitted disposal may be consumed. Acquiring the original lifetime file proves that the actual worker has exited; -missing journal files are not used as proof. Historical reservations without the -binding, generic/binary activities, launched origins and live/replaced locks refuse. - -Confirmation preserves all source files and transcript state. A single checked -paired append acknowledges only that original snapshot activity and records the -request receipt in a Session-creation/request-derived owner aggregate. The writer -rechecks the native proof, original record, Session revision and captured recovery -authority. Retry looks up that durable pair before native access and refuses changed -request content or cross-activity reuse. Cancelling an unstarted recovery request -leaves its original failed restore fenced. Checkout/process uncertainty remains -independent; no Session lease is acknowledged by this operation. This path does -not resolve the origin-before-intent cut or admit a rollback without source proof. +missing journal files are not used as proof. Historical reservations without native +phase proof, generic/binary activities, admitted effects and live/replaced locks +refuse. Old cancellation receipts remain readable without granting new authority. + +Before the first source callback, the worker durably appends source admission to +its native activity, then publishes admission and the exact origin in its private +native phase record, then rechecks authority. Neither marker can be reset to false. +A lost database commit reply prevents callback dispatch, while its durable true +record refuses cancellation. An older database backup cannot override native true +or missing phase evidence. The compatibility migration prevents older binaries +from mutating a Store containing this protocol. The native phase is flushed before +stable lifetime publication, counts toward quota and remains protected from GC. + +Confirmation preserves all source files and transcript state. A checked paired +append acknowledges only that original snapshot activity and records the request +receipt in a Session-creation/request-derived aggregate. The writer rechecks native +proof, original record, Session revision and recovery authority. Retry looks up +that durable pair before native access and refuses changed request content or +cross-activity reuse. A cancellation with an origin then writes a bounded private +`preparation.json` receipt in its exact journal directory. It never synthesizes an +intent or marks an unapplied restore completed. Explicit request acknowledgement +reconciles a missing native receipt after durable Store settlement, retaining exact +worker proof and current authority. Contradictory/corrupt intents refuse; all +original bytes remain retained. Cancelling an unstarted recovery request leaves +its original failed restore fenced. Checkout/process uncertainty stays independent; +no Session lease is acknowledged by this operation. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index e2ea800a1..f6c8eabfd 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -103,7 +103,17 @@ - **WHEN** the owner confirms a fingerprint review of a positive unstarted reservation under its exact reacquired native lifetime lock - **THEN** the writer atomically acknowledges only that activity and retains its cancellation request receipt, rechecking current Session/recovery authority without moving source files or transcript state - **AND** retry looks up the original durable pair without native access, changed or cross-owner request reuse refuses, and a lost commit reply does not dispatch another operation -- **AND** live, legacy, generic, binary or already-launched owners refuse; independent checkout/process uncertainty remains fenced, and cancelling an unstarted resolution leaves its failed parent restore unresolved +- **AND** live, legacy, generic, binary or source-admitted owners refuse; independent checkout/process uncertainty remains fenced, and cancelling an unstarted resolution leaves its failed parent restore unresolved + +#### Scenario: Published origin has no admitted source effects +- **WHEN** an exited worker has a published origin but both its exact database history and version-2 native lifetime phase positively prove no admitted source effects +- **THEN** fingerprint-confirmed cancellation SHALL preserve source files and transcript state, commit the exact paired cancellation receipt, and only then publish bounded native cancellation metadata +- **AND** missing intent alone SHALL NOT grant authority, the original intent SHALL remain unchanged, and explicit acknowledgement SHALL reconcile metadata without replay + +#### Scenario: Database backup cannot erase native source admission +- **WHEN** a database backup shows an unadmitted preparation but its exact native phase is admitted, missing or legacy +- **THEN** preparation cancellation SHALL refuse without changing source files or deleting evidence +- **AND** a lost reply after database source admission SHALL prevent source callback dispatch and SHALL NOT downgrade the durable admitted record #### Scenario: Session changes before a source effect - **WHEN** the Session revision changes after preparation but before the next source mutation diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index f568e8d1d..1974f0ad3 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -162,3 +162,18 @@ retry after restart and admits a subsequent prompt in an unmanaged non-Git Locat Cancelling an unstarted resolution reports `cancelled_before_origin` without acknowledging its original failed restore. Origin-before-intent, independent checkout resources, complete clients, Git, reclamation and the remaining M8 cuts remain open. + + +Pre-effect origin continuation (2026-11-06): new file-copy workers publish version-2 +native source-phase proof before their stable lifetime reservation. A durable +activity admission marker and native true phase precede the first source callback; +missing/legacy phase proof and database/native disagreement refuse cancellation. +The owner cancellation now accepts an exited published origin only with explicit +unadmitted evidence, commits its paired receipt, then publishes bounded journal +cancellation metadata. Explicit preparation acknowledgement reconciles that metadata +without source effects. Original intents and private preparations remain retained. +Local native missing-intent and real-service origin-before-source-admission cuts +are covered; the real-service cut does not assert that intent publication was absent. +Independent checkout resource ownership, Git journaling, cleanup, complete clients +and the remaining M8 cut/platform matrix still prevent acceptance of M2–M8 and +parent tasks 2.4/3.2–3.4. No additional parent task is marked complete. diff --git a/sdk/openapi.json b/sdk/openapi.json index 055e2754d..c1f9485f1 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7441,7 +7441,8 @@ "unknown", "acknowledged", "resolved_by_another_request", - "cancelled_before_origin" + "cancelled_before_origin", + "cancelled_before_effects" ] }, "SnapshotRecoveryReceipt": { @@ -7594,6 +7595,16 @@ }, "warning": { "type": "string" + }, + "origin": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + { + "type": "null" + } + ] } }, "required": [ @@ -7628,6 +7639,12 @@ }, "file_identity": { "type": "string" + }, + "source_phase": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -7719,6 +7736,22 @@ "session_sequence": { "type": "integer", "format": "int64" + }, + "origin": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + { + "type": "null" + } + ] + }, + "source_admitted": { + "type": [ + "boolean", + "null" + ] } }, "required": [ @@ -24866,7 +24899,7 @@ "/api/v1/sessions/{sessionID}/rewind/preparations": { "get": { "operationId": "v1.session.snapshotPreparations", - "summary": "List retained snapshot reservations without a published origin or native access", + "summary": "List retained preparation candidates without native access; review checks eligibility", "tags": [ "session" ], @@ -24964,7 +24997,7 @@ "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/confirm": { "post": { "operationId": "v1.session.snapshotPreparationConfirm", - "summary": "Cancel only a fingerprint-confirmed unstarted snapshot preparation without moving files or transcript", + "summary": "Cancel only a fingerprint-confirmed pre-effect snapshot preparation without moving files or transcript", "tags": [ "session" ], @@ -25085,6 +25118,71 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}/acknowledge": { + "post": { + "operationId": "v1.session.snapshotPreparationAcknowledge", + "summary": "Reconcile native metadata acknowledgement of an exact committed pre-effect cancellation without source changes", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "activityID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "requestID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_SnapshotPreparationReceipt" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/restores/{activityID}/acknowledge": { "post": { "operationId": "v1.session.snapshotRestoreAcknowledge", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index da5997ca0..4436698b6 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1553,6 +1553,12 @@ export interface Operations { body: undefined; response: S.Data_Nullable_Goal; }; + "v1.session.snapshotPreparationAcknowledge": { + path: { sessionID: string; activityID: string; requestID: string }; + query: Record; + body: undefined; + response: S.SnapshotPreparationReceipt; + }; "v1.session.snapshotPreparationConfirm": { path: { sessionID: string; activityID: string }; query: Record; @@ -4692,6 +4698,18 @@ export const operations = { unwrap: false, stream: false, }, + "v1.session.snapshotPreparationAcknowledge": { + tag: "session", + name: "snapshotPreparationAcknowledge", + method: "POST", + path: "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}/acknowledge", + pathParams: ["sessionID", "activityID", "requestID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, "v1.session.snapshotPreparationConfirm": { tag: "session", name: "snapshotPreparationConfirm", @@ -6214,7 +6232,10 @@ export function createGroups(caller: Caller) { /** Clear the current Goal and activate the next queued one; null when the queue was empty (`POST /api/v1/sessions/{sessionID}/goals/skip`) */ skipGoal: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.skipGoal", { path: { sessionID } }, options), - /** Cancel only a fingerprint-confirmed unstarted snapshot preparation without moving files or transcript (`POST /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/confirm`) */ + /** Reconcile native metadata acknowledgement of an exact committed pre-effect cancellation without source changes (`POST /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}/acknowledge`) */ + snapshotPreparationAcknowledge: (sessionID: string, activityID: string, requestID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotPreparationAcknowledge", { path: { sessionID, activityID, requestID } }, options), + /** Cancel only a fingerprint-confirmed pre-effect snapshot preparation without moving files or transcript (`POST /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/confirm`) */ snapshotPreparationConfirm: (sessionID: string, activityID: string, body: Operations["v1.session.snapshotPreparationConfirm"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.snapshotPreparationConfirm", { path: { sessionID, activityID }, body }, options), /** Look up the original durable preparation cancellation pair without native access or replay (`GET /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}`) */ @@ -6223,7 +6244,7 @@ export function createGroups(caller: Caller) { /** Review cancellation of an exited preparation using its exact native lifetime proof (`POST /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/review`) */ snapshotPreparationReview: (sessionID: string, activityID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotPreparationReview", { path: { sessionID, activityID } }, options), - /** List retained snapshot reservations without a published origin or native access (`GET /api/v1/sessions/{sessionID}/rewind/preparations`) */ + /** List retained preparation candidates without native access; review checks eligibility (`GET /api/v1/sessions/{sessionID}/rewind/preparations`) */ snapshotPreparations: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotPreparations", { path: { sessionID } }, options), /** Inspect retained filesystem restore journals without acknowledging or replaying native effects (`GET /api/v1/sessions/{sessionID}/rewind/recovery`) */ diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 58b7f036f..5de2c677f 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -4726,6 +4726,7 @@ export type SnapshotLifetimeBinding = { namespace_identity: string; directory_identity: string; file_identity: string; + source_phase?: string | null; }; export type SnapshotLifetimeRequest = { @@ -4763,6 +4764,8 @@ export type SnapshotPreparationReceipt = { lifetime: SnapshotLifetimeBinding; native_sequence: number; session_sequence: number; + origin?: SnapshotOrigin | null; + source_admitted?: boolean | null; }; export type SnapshotPreparationReview = { @@ -4772,6 +4775,7 @@ export type SnapshotPreparationReview = { session_sequence: number; confirmation: string; warning: string; + origin?: SnapshotOrigin | null; }; /** Retained filesystem evidence; completion does not acknowledge a Session/native receipt. */ @@ -4865,7 +4869,7 @@ export type SnapshotRecoverySelection = { action: SnapshotRecoveryAction; }; -export type SnapshotRecoveryState = "pending" | "unknown" | "acknowledged" | "resolved_by_another_request" | "cancelled_before_origin"; +export type SnapshotRecoveryState = "pending" | "unknown" | "acknowledged" | "resolved_by_another_request" | "cancelled_before_origin" | "cancelled_before_effects"; /** * Exact native intent and Session batch consumed by an ordinary restore. diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index 13882edeb..91f15efbf 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -89,3 +89,14 @@ test("preparation recovery lists, reviews, confirms and looks up an exact cancel assert.deepEqual(JSON.parse(calls[2]!.body!), confirmation); assert.equal(calls[3]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/preparations/op_fixture/requests/cancel%2Fwith%20space"); }); + +test("pre-effect cancellation acknowledgement retains origin proof without replay input", async () => { + const receipt = { activity_id: "op_fixture", request_id: "cancel/with space", + origin: { activity_id: "op_fixture" }, source_admitted: false, + lifetime: { version: 2, source_phase: "phase_fixture" } }; + const { client, calls } = mockClient(() => json(200, { data: receipt })); + assert.deepEqual(await client.session.snapshotPreparationAcknowledge("ses_owner", "op_fixture", receipt.request_id), receipt); + assert.equal(calls[0]!.method, "POST"); + assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/preparations/op_fixture/requests/cancel%2Fwith%20space/acknowledge"); + assert.ok(!calls[0]!.body); +}); From 3388dc914526131ee12c230fe8399a6b74a87a6c Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 22:18:07 -0400 Subject: [PATCH 13/87] Give snapshot workers independent checkout claims --- crates/cyber-app/src/host.rs | 9 ++ crates/cyber-core/src/worktrees/activity.rs | 14 +++ .../tests/worktree_repository/activity.rs | 97 +++++++++++++++++ crates/cyber-server/src/runtime/host.rs | 16 +++ crates/cyber-server/src/runtime/location.rs | 20 ++++ .../cyber-server/src/runtime/snapshot_work.rs | 4 +- .../src/runtime/snapshot_work_tests.rs | 101 ++++++++++++++++++ crates/cyber-tools/src/host.rs | 9 ++ crates/cyber-tools/src/worktrees.rs | 51 +++++++-- crates/cyber-tools/tests/worktree_setup.rs | 46 ++++++++ docs/goal-delivery.md | 7 ++ docs/rewind.md | 9 ++ .../implementation.md | 52 +++++++++ .../managed-recovery.md | 21 ++++ .../specs/snapshots-checkpoints/spec.md | 6 ++ .../complete-daily-coding-surfaces/tasks.md | 12 +++ 16 files changed, 462 insertions(+), 12 deletions(-) diff --git a/crates/cyber-app/src/host.rs b/crates/cyber-app/src/host.rs index 50027b087..48e5299a4 100644 --- a/crates/cyber-app/src/host.rs +++ b/crates/cyber-app/src/host.rs @@ -270,6 +270,15 @@ impl ToolHost for AppHost { self.builtin.claim_binary_location(info, document, cancel) } + fn claim_snapshot_location<'a>( + &'a self, + info: &'a cyber_server::runtime::SessionInfo, + activity: &'a str, + cancel: CancellationToken, + ) -> BoxFuture<'a, Result> { + self.builtin.claim_snapshot_location(info, activity, cancel) + } + fn prepare_child_continuation<'a>( &'a self, parent: &'a cyber_server::runtime::SessionInfo, diff --git a/crates/cyber-core/src/worktrees/activity.rs b/crates/cyber-core/src/worktrees/activity.rs index 117c101e5..df7050012 100644 --- a/crates/cyber-core/src/worktrees/activity.rs +++ b/crates/cyber-core/src/worktrees/activity.rs @@ -157,6 +157,18 @@ impl Repository { self.claim_owner(execution, managed, document).await } + /// A synchronous snapshot worker owns checkout activity independently of the + /// Session and of every other worker. An interrupted claim remains unknown. + pub async fn claim_snapshot( + &self, + execution: &dyn GitExecution, + managed: &Managed, + activity: &str, + ) -> io::Result { + safe_id(activity, "op")?; + self.claim_owner(execution, managed, activity).await + } + async fn claim_owner( &self, execution: &dyn GitExecution, @@ -229,6 +241,8 @@ fn safe_owner_id(id: &str) -> io::Result<()> { "lsp" } else if crate::ids::has_prefix(id, "bin") { "bin" + } else if crate::ids::has_prefix(id, "op") { + "op" } else { "ses" }; diff --git a/crates/cyber-core/tests/worktree_repository/activity.rs b/crates/cyber-core/tests/worktree_repository/activity.rs index 195f14384..a69c03263 100644 --- a/crates/cyber-core/tests/worktree_repository/activity.rs +++ b/crates/cyber-core/tests/worktree_repository/activity.rs @@ -1,6 +1,103 @@ use super::*; use cyber_core::worktrees::CheckoutActivity; +#[test] +fn snapshot_worker_owns_a_distinct_checkout_lock_through_retained_settlement() { + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("snapshot-worker").unwrap(), + &Settings::default(), + ) + .unwrap(); + let actor = block_on(repository.claim(&fixture.execution, &managed, "ses_actor")).unwrap(); + for foreign in ["ses_actor", "bin_worker", "../op_worker"] { + assert!( + block_on(repository.claim_snapshot(&fixture.execution, &managed, foreign)).is_err() + ); + } + assert!(block_on(repository.claim(&fixture.execution, &managed, "op_worker")).is_err()); + assert!(block_on(repository.claim_binary(&fixture.execution, &managed, "op_worker")).is_err()); + let worker = + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")).unwrap(); + assert_eq!(worker.owner_id(), "op_worker"); + assert!( + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")).is_err() + ); + actor.settle().unwrap(); + let proof = worker.settle_retained().unwrap(); + let error = block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)) + .unwrap_err(); + assert!(error.to_string().contains("op_worker"), "{error}"); + drop(proof); + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)).unwrap(); +} + +#[test] +fn snapshot_claim_disposal_retains_unknown_evidence_without_reusing_a_session_claim() { + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("snapshot-unknown").unwrap(), + &Settings::default(), + ) + .unwrap(); + let worker = + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")).unwrap(); + drop(worker); + let record = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id) + .join("op_worker.lock"); + let retained = std::fs::read(&record).unwrap(); + assert!( + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")).is_err() + ); + block_on(repository.claim(&fixture.execution, &managed, "ses_actor")) + .unwrap() + .settle() + .unwrap(); + let error = block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)) + .unwrap_err(); + assert!(error.to_string().contains("unknown"), "{error}"); + assert_eq!(std::fs::read(record).unwrap(), retained); +} + +#[test] +fn settled_snapshot_worker_does_not_acknowledge_unknown_session_checkout_activity() { + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("snapshot-session").unwrap(), + &Settings::default(), + ) + .unwrap(); + drop(block_on(repository.claim(&fixture.execution, &managed, "ses_actor")).unwrap()); + let record = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id) + .join("ses_actor.lock"); + let retained = std::fs::read(&record).unwrap(); + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")) + .unwrap() + .settle() + .unwrap(); + assert!(block_on(repository.claim(&fixture.execution, &managed, "ses_actor")).is_err()); + assert!( + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)) + .is_err() + ); + assert_eq!(std::fs::read(record).unwrap(), retained); +} + #[test] fn binary_worker_retains_an_independent_checkout_lease_through_source_commit() { let fixture = Fixture::new(); diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 98e7673b6..64ba1756d 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -854,6 +854,22 @@ pub trait ToolHost: Send + Sync { Box::pin(async { Err("Binary worker checkout ownership is unsupported".into()) }) } + /// Retain a snapshot worker's checkout independently of any Session claim. + /// Managed hosts must implement this explicitly; the generic Session claim + /// cannot supply exact snapshot ownership. + fn claim_snapshot_location<'a>( + &'a self, + info: &'a SessionInfo, + _activity: &'a str, + cancel: CancellationToken, + ) -> BoxFuture<'a, Result> { + if info.worktree_id.is_some() { + Box::pin(async { Err("Snapshot worker checkout ownership is unsupported".into()) }) + } else { + self.claim_location(info, false, cancel) + } + } + /// Prepare an existing child under its exclusive owner. Opening a client view grants no bypass. fn prepare_child_continuation<'a>( &'a self, diff --git a/crates/cyber-server/src/runtime/location.rs b/crates/cyber-server/src/runtime/location.rs index 5ef210a98..225257fd3 100644 --- a/crates/cyber-server/src/runtime/location.rs +++ b/crates/cyber-server/src/runtime/location.rs @@ -16,6 +16,26 @@ impl Inner { let claim = self .tools .claim_location(info, creating, cancel.child_token()); + self.await_location_claim(claim, cancel).await + } + + pub(super) async fn claim_snapshot_location( + &self, + info: &SessionInfo, + activity: &str, + cancel: CancellationToken, + ) -> Result { + let claim = self + .tools + .claim_snapshot_location(info, activity, cancel.child_token()); + self.await_location_claim(claim, cancel).await + } + + async fn await_location_claim( + &self, + claim: impl Future>, + cancel: CancellationToken, + ) -> Result { tokio::pin!(claim); let result = tokio::select! { result = &mut claim => result, diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 19d79d720..962221c4c 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -379,7 +379,7 @@ impl Runtime { )?; let lease = self .inner - .claim_location(&state.info, false, scope.control.stop.clone()) + .claim_snapshot_location(&state.info, scope.id(), scope.control.stop.clone()) .await?; let directory = std::fs::canonicalize(&state.info.directory) .map_err(|error| RuntimeError::Invalid(error.to_string()))?; @@ -439,7 +439,7 @@ impl Runtime { .await?; let lease = self .inner - .claim_location(&state.info, false, scope.control.stop.clone()) + .claim_snapshot_location(&state.info, scope.id(), scope.control.stop.clone()) .await?; if lease.worktree_id != state.info.worktree_id { return Err(RuntimeError::Conflict( diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 022b3b6a5..3a37cc50b 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -88,6 +88,107 @@ fn runtime_with_snapshots( }) } +#[derive(Default)] +struct SnapshotClaims { + ordinary_forbidden: std::sync::atomic::AtomicBool, + claimed: std::sync::Mutex>, +} +impl super::super::ToolHost for SnapshotClaims { + fn definitions(&self, _: &super::super::TurnContext) -> Vec { + vec![] + } + fn execute( + &self, + _: super::super::Invocation, + _: tokio_util::sync::CancellationToken, + ) -> futures::future::BoxFuture<'_, super::super::ToolOutcome> { + Box::pin(async { super::super::ToolOutcome::Failed("unused fixture tool".into()) }) + } + fn claim_location<'a>( + &'a self, + _: &'a super::super::SessionInfo, + _: bool, + _: tokio_util::sync::CancellationToken, + ) -> futures::future::BoxFuture<'a, Result> { + Box::pin(async move { + if self.ordinary_forbidden.load(Ordering::SeqCst) { + return Err("Snapshot reused the ordinary Session claim".into()); + } + Ok(super::super::LocationLease::unmanaged()) + }) + } + fn claim_snapshot_location<'a>( + &'a self, + _: &'a super::super::SessionInfo, + activity: &'a str, + _: tokio_util::sync::CancellationToken, + ) -> futures::future::BoxFuture<'a, Result> { + Box::pin(async move { + self.claimed.lock().unwrap().push(activity.into()); + Ok(super::super::LocationLease::unmanaged()) + }) + } +} + +#[tokio::test] +async fn snapshot_location_port_receives_the_reserved_native_activity_instead_of_the_session() { + let (temp, base, _, _, _) = super::super::binary_documents_tests::fixture().await; + let host = Arc::new(SnapshotClaims::default()); + let runtime = Runtime::new(RuntimeOptions { + store: base.inner.store.clone(), + resolver: base.inner.resolver.clone(), + tools: host.clone(), + global_config_dir: temp.path().join("config"), + shell: "test".into(), + claude_compat: false, + compaction: Default::default(), + retry: Default::default(), + max_steps: None, + today: None, + interactive: false, + snapshots: Arc::new(super::super::NoSnapshots), + }); + let session = runtime + .create_session(super::super::CreateSession { + directory: temp.path().to_str().unwrap().into(), + model: "test/main".into(), + ..Default::default() + }) + .await + .unwrap(); + host.ordinary_forbidden.store(true, Ordering::SeqCst); + let owner = worker(&runtime, &session.id).await; + assert_eq!(*host.claimed.lock().unwrap(), [owner.scope.id().to_owned()]); + let record = latest(&runtime, &session.id); + assert_eq!(record["snapshot_worker"], true); + assert_eq!(record["session_id"], session.id); + assert_eq!(record["phase"], "reserved"); + drop(owner); + runtime.shutdown().await; + base.shutdown().await; +} + +#[tokio::test] +async fn default_host_refuses_managed_snapshot_claim_without_explicit_support() { + use super::super::ToolHost; + let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let mut info = runtime.session_info(&session).await.unwrap(); + info.worktree_id = Some("wt_fixture".into()); + let host = super::super::NoTools; + assert!( + host.claim_snapshot_location(&info, "op_fixture", Default::default()) + .await + .is_err() + ); + info.worktree_id = None; + host.claim_snapshot_location(&info, "op_fixture", Default::default()) + .await + .unwrap() + .settle() + .unwrap(); + runtime.shutdown().await; +} + struct Acknowledgements(Arc); struct Lifetimes { inner: Acknowledgements, diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index d26913517..4b65949d7 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1353,6 +1353,15 @@ impl ToolHost for BuiltinHost { Box::pin(self.claim_binary_worktree_location(info, document, cancel)) } + fn claim_snapshot_location<'a>( + &'a self, + info: &'a cyber_server::runtime::SessionInfo, + activity: &'a str, + cancel: CancellationToken, + ) -> BoxFuture<'a, Result> { + Box::pin(self.claim_snapshot_worktree_location(info, activity, cancel)) + } + fn prepare_child_continuation<'a>( &'a self, parent: &'a cyber_server::runtime::SessionInfo, diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index a2adf1e29..51bc79468 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -32,6 +32,12 @@ use crate::host::{BuiltinHost, Ctx}; use crate::permissions::Request; use crate::tools::process::Process; +#[derive(Clone, Copy)] +enum CheckoutWorker<'a> { + Binary(&'a str), + Snapshot(&'a str), +} + /// Inputs for creating a new Session in a managed worktree. pub struct WorktreeSessionRequest { pub session: CreateSession, @@ -219,7 +225,24 @@ impl BuiltinHost { info.worktree_id.as_deref(), false, cancel, - Some(document), + Some(CheckoutWorker::Binary(document)), + ) + .await + } + + pub(crate) async fn claim_snapshot_worktree_location( + &self, + info: &SessionInfo, + activity: &str, + cancel: CancellationToken, + ) -> Result { + let inv = location_invocation(info); + self.claim_location( + &inv, + info.worktree_id.as_deref(), + false, + cancel, + Some(CheckoutWorker::Snapshot(activity)), ) .await } @@ -259,7 +282,7 @@ impl BuiltinHost { bound: Option<&str>, creating: bool, cancel: CancellationToken, - binary: Option<&str>, + worker: Option>, ) -> Result { use cyber_server::runtime::LocationLease; let activity = self @@ -308,14 +331,22 @@ impl BuiltinHost { for (repository, managed) in locations { leases.push( retry_repository_busy(&ctx.cancel, || async { - if let Some(document) = binary { - repository - .claim_binary(&execution, &managed, document) - .await - } else { - repository - .claim(&execution, &managed, &inv.session_id) - .await + match worker { + Some(CheckoutWorker::Binary(document)) => { + repository + .claim_binary(&execution, &managed, document) + .await + } + Some(CheckoutWorker::Snapshot(activity)) => { + repository + .claim_snapshot(&execution, &managed, activity) + .await + } + None => { + repository + .claim(&execution, &managed, &inv.session_id) + .await + } } }) .await diff --git a/crates/cyber-tools/tests/worktree_setup.rs b/crates/cyber-tools/tests/worktree_setup.rs index bd083fc47..02eb200c8 100644 --- a/crates/cyber-tools/tests/worktree_setup.rs +++ b/crates/cyber-tools/tests/worktree_setup.rs @@ -1912,6 +1912,52 @@ async fn process_launch_failure_remains_uncertain_and_cannot_authorize_retry() { assert!(!managed.path.join("target.txt").exists()); } +#[tokio::test] +async fn snapshot_location_claim_is_independent_of_a_live_session_and_retained_until_disposal() { + use cyber_core::worktrees::CheckoutActivity; + use cyber_server::runtime::{CreateSession, NoSnapshots, ToolHost}; + let (fixture, repository, managed) = owned().await; + #[cfg(windows)] + fixture.set_config(json!({"sandbox":{"policy":"full-access"}})); + let flow = support::flow::Flow::with(fixture, vec![], false, Arc::new(NoSnapshots)); + let info = flow + .runtime + .create_session(CreateSession { + directory: managed.path.display().to_string(), + model: "test/main".into(), + mode: Some("bypass".into()), + ..Default::default() + }) + .await + .unwrap(); + let actor = flow + .f + .host + .claim_location(&info, false, CancellationToken::new()) + .await + .unwrap(); + let worker = flow + .f + .host + .claim_snapshot_location(&info, "op_snapshot", CancellationToken::new()) + .await + .unwrap(); + assert_eq!(worker.worktree_id.as_deref(), Some(managed.id.as_str())); + actor.settle().unwrap(); + let proof = worker.settle_retained().unwrap(); + let error = repository + .remove(&Git, &CheckoutActivity, &managed, false) + .await + .unwrap_err(); + assert!(error.to_string().contains("op_snapshot"), "{error}"); + drop(proof); + repository + .remove(&Git, &CheckoutActivity, &managed, false) + .await + .unwrap(); + flow.runtime.shutdown().await; +} + #[tokio::test] async fn retained_native_location_proof_prevents_checkout_removal_until_disposal() { use cyber_core::worktrees::CheckoutActivity; diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 1cbf838c7..7849e63f2 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -157,3 +157,10 @@ must both permit cancellation; a database backup cannot erase native admission. Original intents and payloads remain retained. This is progress on the approved recovery protocol, with independent checkout ownership, Git journaling, safe cleanup, complete clients and full platform/cut acceptance still required before delivery. + + +The next local ownership continuation moves snapshot and recovery checkout claims +from Session IDs to independent native activity IDs through a dedicated host port. +CLI/App retain those locks through settlement; unsupported managed hosts refuse. +Unknown checkout claims remain protected. Exact interrupted-claim proof and +reconciliation are still required before accepting managed resource recovery. diff --git a/docs/rewind.md b/docs/rewind.md index 3891f9dfa..db5a4e3fd 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -287,3 +287,12 @@ preparations remain protected for acknowledged safe cleanup. Do not edit records or remove evidence to force a retry. Independent managed-checkout proof, Git journaling, reclamation, typed recovery clients and broader native crash/platform acceptance remain unfinished. + + +Managed checkout snapshots use an independent claim named by the snapshot's native +activity, so they do not reuse a Session, binary document, MCP or language-server +claim. The native claim remains locked through worker settlement. Hosts without +explicit support refuse managed snapshot admission. A crash while acquiring a claim +can still leave unknown checkout evidence: preparation cancellation does not clear +that evidence or authorize checkout removal. Exact native claim reconciliation is +still required for that acquisition cut; retain the evidence for recovery. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 0318c12ce..109ea4693 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1175,3 +1175,55 @@ claimed. The next ownership gap is the managed checkout acquisition/settlement c its Session-owned claim must become independent exact worker evidence before an exited preparation can safely settle that resource. No existing unknown lease may be cleared merely because the snapshot worker exited. + + +### Independent checkout acquisition for snapshot workers (2026-11-06, local) + +The prior code retained a checkout lease but acquired it under the Session ID. +Snapshot restoration and keep/rollback workers now use a dedicated +`claim_snapshot_location` port with their already-reserved native activity ID. +The core repository issues a distinct `op_` checkout claim; the production tools +adapter applies it to every enclosing managed checkout, and the App adapter +forwards this port. The existing retained settlement holds each native lock and +Location activity until the actual worker releases its resources. Generic Session, +binary, MCP and LSP claim selection is unchanged. A managed host without this +explicit port refuses; unmanaged defaults retain existing Location behavior. + +Core regressions run real Git checkout creation/removal: Session/snapshot claims +coexist, duplicate snapshot ownership refuses, generic/binary/foreign IDs cannot +substitute, retained settlement excludes removal, interrupted acquisition stays +unknown, and snapshot settlement cannot acknowledge an unknown Session. A production +host regression acquires a snapshot alongside a live Session, retains its exact +activity-named lock, then permits removal only after proof disposal. Runtime tests +verify dispatch of the actual reserved activity and the default managed-host refusal. + +This does not create acknowledgement authority from an activity name. Exact +persisted checkout file/directory identity, creation/lifetime binding, multi-checkout +acquisition proof and interrupted-claim reconciliation remain required. No unknown +Session or operation record is force-cleared. The complete managed recovery delivery +and both issues remain incomplete; no parent task is marked complete here. + + +Local macOS arm64 verification for this continuation: all 68 core worktree repository +cases passed, with its existing subprocess fixture listed ignored; all 418 server +unit cases passed with one ignored fixture, plus 22 affected integrations (4 idle +cancellation, 13 Location activity, 5 rewind preview). All 38 production worktree +host cases and all 16 real CLI branch/recovery cases passed, including retained +native ownership, service kills, early preparations and partial restore keep/rollback. +All-target core/server/tools/App/CLI Clippy passed with warnings denied. Formatting +and diff checks passed. Strict pinned OpenSpec passed all 117 items; lint reports +0 errors/25 warnings. No HTTP payload or generated SDK contract changed. Full-workspace, +Linux/Windows native execution and live TUI/App recovery acceptance were not run. +Observed native sandbox deprecation and Darwin linker warnings are recorded without +assigning their origin to main. + +The code-review skill was applied across all changed core/runtime/host/App/test and +documentation files. Review confirms the port/adapters fit the existing architecture, +typed worker claims cannot reuse Session/binary namespaces, source/worktree binding +checks remain in the runtime, ordinary cancellation acknowledgement behavior is +shared unchanged, and the actual retained Location resource continues to outlive +filesystem effects and consuming commits. Tests cover the new public core and host +ports and refusal paths. No critical/major findings remain for independent acquisition; +immutable interrupted-claim proof and full managed recovery acceptance remain open. +This continuation remains local and does not authorize unknown claim settlement, +complete M2–M8 or make the full recovery increment merge-ready. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 020ca2a15..3da5f9557 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -280,3 +280,24 @@ worker proof and current authority. Contradictory/corrupt intents refuse; all original bytes remain retained. Cancelling an unstarted recovery request leaves its original failed restore fenced. Checkout/process uncertainty stays independent; no Session lease is acknowledged by this operation. + + +### Independent checkout claim acquisition + +Snapshot workers and recovery-resolution workers use a dedicated host admission +port carrying the already-reserved native activity ID. The production CLI/App +adapter claims each managed checkout under that activity, independently of the +Session, binary documents, LSP and MCP owners. Retained settlement continues to +hold every actual checkout descriptor and Location activity through worker teardown. +A managed host that has not implemented this port refuses rather than falling back +to ordinary Session admission. Unmanaged hosts preserve their existing Location +activity behavior through the default adapter. + +An interrupted checkout acquisition may retain an unknown activity-owned lease. +The immutable lifetime reservation precedes that acquisition, but the activity ID +alone is not a native acknowledgement receipt. This continuation does not clear +such records, infer missing claims from an empty directory or acknowledge another +Session. Exact persisted checkout file/directory identity and creation/lifetime +binding, consuming proof through Store settlement, reconciliation and real kill +cuts across multi-checkout acquisition remain required before the complete resource +recovery path is accepted. Older Session-owned unknown leases remain independent. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index f6c8eabfd..b927117cf 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -93,6 +93,12 @@ - **THEN** its actual native owners remain held until the synchronous worker exits, Stop and read-only Session inspection remain available between source effects, and shutdown waits for actual ownership settlement - **AND** uncertain effects retain their origin and admission fence across runtime reload without replay +#### Scenario: Snapshot checkout claim is independent of the Session +- **WHEN** a snapshot worker acquires a managed checkout while its Session or another worker owns a separate claim +- **THEN** checkout admission SHALL use the exact reserved native activity ID, retain its own lock through consuming settlement, and refuse reuse of a live or unknown claim +- **AND** settling that snapshot claim SHALL NOT acknowledge unrelated Session activity; an interrupted acquisition SHALL retain unknown evidence without a generic clearing path +- **AND** a managed host without explicit snapshot ownership support SHALL refuse instead of reusing a Session claim + #### Scenario: Lifetime proof precedes reservation - **WHEN** a new file-copy worker reserves runtime ownership before publishing its restore origin - **THEN** an immutable worker-specific native lifetime lock already binds its activity, Session creation, checkout, Location and private namespace, and remains retained through actual resource teardown diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 1974f0ad3..b88d64e87 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -177,3 +177,15 @@ are covered; the real-service cut does not assert that intent publication was ab Independent checkout resource ownership, Git journaling, cleanup, complete clients and the remaining M8 cut/platform matrix still prevent acceptance of M2–M8 and parent tasks 2.4/3.2–3.4. No additional parent task is marked complete. + + +Independent checkout acquisition continuation: ordinary snapshot and recovery +workers now call a dedicated host port with their already-reserved activity ID. +The production CLI/App adapters retain separate `op_` checkout locks alongside +Session/binary/MCP/LSP claims. Managed hosts without explicit support refuse. +Retained settlement still holds actual descriptors through teardown. Regression +coverage verifies coexistence, duplicate/foreign ID refusal, removal exclusion, +unknown claim retention and preservation of unrelated Session evidence. This is +acquisition ownership, not native reconciliation authority: immutable persisted +checkout route/creation/lifetime binding and interrupted-claim acknowledgement +remain incomplete, so M2–M8 and parent tasks remain open. From 7e47017bec0ce259869b059ee8230ed31e4471e3 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 23:05:55 -0400 Subject: [PATCH 14/87] Verify retained snapshot checkout proof at consuming commits --- crates/cyber-core/src/worktrees.rs | 2 +- crates/cyber-core/src/worktrees/activity.rs | 67 ++++- .../src/worktrees/activity/snapshot.rs | 281 ++++++++++++++++++ .../tests/worktree_repository/activity.rs | 105 +++++++ crates/cyber-server/src/runtime/activity.rs | 117 +++++--- crates/cyber-server/src/runtime/host.rs | 29 ++ crates/cyber-server/src/runtime/mod.rs | 6 +- .../cyber-server/src/runtime/snapshot_work.rs | 50 +++- .../src/runtime/snapshot_work_tests.rs | 183 ++++++++++++ ...20261106020000_snapshot_checkout_proof.sql | 3 + crates/cyber-store/src/migrate.rs | 43 ++- crates/cyber-tools/src/worktrees.rs | 44 +++ crates/cyber-tools/tests/worktree_setup.rs | 94 +++++- docs/goal-delivery.md | 7 + docs/rewind.md | 7 + .../implementation.md | 85 ++++++ .../managed-recovery.md | 31 ++ .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 9 + 19 files changed, 1108 insertions(+), 62 deletions(-) create mode 100644 crates/cyber-core/src/worktrees/activity/snapshot.rs create mode 100644 crates/cyber-store/migrations/20261106020000_snapshot_checkout_proof.sql diff --git a/crates/cyber-core/src/worktrees.rs b/crates/cyber-core/src/worktrees.rs index f0939fea6..e627876b2 100644 --- a/crates/cyber-core/src/worktrees.rs +++ b/crates/cyber-core/src/worktrees.rs @@ -8,7 +8,7 @@ mod location; mod removal; pub(crate) mod repository; mod setup; -pub use activity::{CheckoutActivity, CheckoutLease}; +pub use activity::{CheckoutActivity, CheckoutBinding, CheckoutLease}; pub use changes::{ChangedFile, Changes}; pub use includes::IncludedFile; pub use removal::{RemovalActivity, RemovalPhase, RemovalRecord}; diff --git a/crates/cyber-core/src/worktrees/activity.rs b/crates/cyber-core/src/worktrees/activity.rs index df7050012..7d1dfa4ee 100644 --- a/crates/cyber-core/src/worktrees/activity.rs +++ b/crates/cyber-core/src/worktrees/activity.rs @@ -7,13 +7,17 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use super::{GitExecution, Managed, Name, RemovalActivity, Repository, RepositoryLock}; +mod snapshot; +pub use snapshot::CheckoutBinding; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct UseRecord { worktree_id: String, #[serde(alias = "session_id")] owner_id: String, settled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + snapshot_binding: Option, } /// Dropping without `settle` releases the OS lock but retains unknown activity. @@ -21,6 +25,7 @@ struct UseRecord { pub struct CheckoutLease { file: LockedRecord, record: UseRecord, + snapshot_pins: Option, } impl CheckoutLease { @@ -32,14 +37,36 @@ impl CheckoutLease { &self.record.owner_id } + pub fn snapshot_binding(&self) -> io::Result<&CheckoutBinding> { + self.record + .snapshot_binding + .as_ref() + .ok_or_else(|| invalid("Checkout has no exact snapshot binding")) + } + + /// Metadata-only verification against retained native descriptors. This does + /// not acknowledge an unknown lease or launch Git/process work. + pub fn verify_snapshot(&self) -> io::Result<()> { + self.snapshot_pins + .as_ref() + .ok_or_else(|| invalid("Checkout snapshot pins are missing"))? + .verify(&self.file, &self.record) + } + pub fn settle(self) -> io::Result<()> { self.settle_retained().map(drop) } /// Record acknowledgement while retaining the native lock through a later commit. pub fn settle_retained(mut self) -> io::Result { + if self.record.snapshot_binding.is_some() { + self.verify_snapshot()?; + } self.record.settled = true; write_record(&mut self.file, &self.record)?; + if self.record.snapshot_binding.is_some() { + self.verify_snapshot()?; + } Ok(self) } } @@ -166,7 +193,13 @@ impl Repository { activity: &str, ) -> io::Result { safe_id(activity, "op")?; - self.claim_owner(execution, managed, activity).await + let mut lease = self.claim_owner(execution, managed, activity).await?; + let (binding, pins) = snapshot::Pins::capture(managed, activity, &lease.file)?; + lease.record.snapshot_binding = Some(binding); + lease.snapshot_pins = Some(pins); + write_record(&mut lease.file, &lease.record)?; + lease.verify_snapshot()?; + Ok(lease) } async fn claim_owner( @@ -199,7 +232,17 @@ impl Repository { } let path = use_directory(managed, true)?.join(format!("{session}.lock")); let existed = path.try_exists()?; - let file = open_record(&path, true)?; + let file = if crate::ids::has_prefix(session, "op") { + // An activity is a one-shot owner. Preserve old/future evidence even + // after settlement; no new operation may reset this claim. + OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path)? + } else { + open_record(&path, true)? + }; if !try_lock(&file)? { return Err(io::Error::other(format!("Worktree in use by {session}"))); } @@ -215,10 +258,15 @@ impl Repository { worktree_id: managed.id.clone(), owner_id: session.into(), settled: false, + snapshot_binding: None, }; write_record(&mut file, &record)?; sync_parent(&path)?; - Ok(CheckoutLease { file, record }) + Ok(CheckoutLease { + file, + record, + snapshot_pins: None, + }) } } @@ -315,6 +363,9 @@ fn check_record(record: &UseRecord, managed: &Managed, session: &str) -> io::Res fn write_record(file: &mut File, record: &UseRecord) -> io::Result<()> { let bytes = serde_json::to_vec(record).map_err(invalid)?; + if bytes.len() > 4096 { + return Err(invalid("Activity record exceeds 4 KiB")); + } file.rewind()?; file.set_len(0)?; file.write_all(&bytes)?; @@ -364,11 +415,13 @@ mod tests { worktree_id: "wt_test".into(), owner_id: "ses_test".into(), settled: false, + snapshot_binding: None, }; write_record(&mut file, &record).unwrap(); let proof = CheckoutLease { file: LockedRecord(file), record, + snapshot_pins: None, } .settle_retained() .unwrap(); @@ -397,6 +450,7 @@ mod tests { worktree_id: "wt_test".into(), owner_id: "ses_test".into(), settled: false, + snapshot_binding: None, }; write_record(&mut initial, &record).unwrap(); drop(initial); @@ -405,7 +459,8 @@ mod tests { assert!( CheckoutLease { file: LockedRecord(file), - record + record, + snapshot_pins: None } .settle_retained() .is_err() @@ -449,11 +504,13 @@ mod tests { worktree_id: "wt_test".into(), owner_id: "ses_test".into(), settled: false, + snapshot_binding: None, }; write_record(&mut file, &record).unwrap(); let lease = CheckoutLease { file: LockedRecord(file), record, + snapshot_pins: None, }; if settle { lease.settle().unwrap(); diff --git a/crates/cyber-core/src/worktrees/activity/snapshot.rs b/crates/cyber-core/src/worktrees/activity/snapshot.rs new file mode 100644 index 000000000..805067a49 --- /dev/null +++ b/crates/cyber-core/src/worktrees/activity/snapshot.rs @@ -0,0 +1,281 @@ +//! Exact checkout metadata pins. A binding alone never grants acknowledgement. +use super::{Managed, UseRecord, invalid}; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs::File, + io, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckoutBinding { + pub version: u32, + pub owner_id: String, + pub worktree_id: String, + pub name: String, + pub common_dir: PathBuf, + pub common_identity: String, + pub root_identity: String, + pub directory_identity: String, + pub file_identity: String, + pub creation_directory_identity: String, + pub creation_file_identity: String, + pub creation_digest: String, +} + +pub(super) struct Pins { + common: Dir, + root: Dir, + directory: Dir, + creations: Dir, + creation: File, +} + +fn open_file(directory: &Dir, name: &str) -> io::Result { + let file = directory + .open_with( + name, + OpenOptions::new().read(true).follow(FollowSymlinks::No), + )? + .into_std(); + regular(&file)?; + Ok(file) +} + +fn regular(file: &File) -> io::Result<()> { + if !file.metadata()?.is_file() { + return Err(invalid("Snapshot checkout record is not regular")); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if file.metadata()?.nlink() != 1 { + return Err(invalid("Snapshot checkout record has aliases")); + } + } + #[cfg(windows)] + windows::regular(file)?; + Ok(()) +} + +#[cfg(unix)] +fn identity(file: &File) -> io::Result { + use std::os::unix::fs::MetadataExt; + let metadata = file.metadata()?; + let birth = metadata + .created()? + .duration_since(std::time::UNIX_EPOCH) + .map_err(invalid)?; + Ok(format!( + "{}:{}:{}:{}:{}:{}:{}", + metadata.dev(), + metadata.ino(), + birth.as_secs(), + birth.subsec_nanos(), + metadata.uid(), + metadata.gid(), + metadata.mode() + )) +} +#[cfg(windows)] +fn identity(file: &File) -> io::Result { + windows::identity(file) +} +#[cfg(not(any(unix, windows)))] +fn identity(_: &File) -> io::Result { + Err(invalid( + "Native snapshot checkout identities are unavailable", + )) +} + +fn directory_identity(directory: &Dir) -> io::Result { + identity(&directory.try_clone()?.into_std_file()) +} +fn bytes(file: &File) -> io::Result> { + contents(file, 1024 * 1024) +} +fn contents(file: &File, limit: usize) -> io::Result> { + let mut result = Vec::new(); + let mut chunk = [0; 4096]; + loop { + let capacity = chunk.len().min(limit + 1 - result.len()); + #[cfg(unix)] + let read = + std::os::unix::fs::FileExt::read_at(file, &mut chunk[..capacity], result.len() as u64)?; + #[cfg(windows)] + let read = std::os::windows::fs::FileExt::seek_read( + file, + &mut chunk[..capacity], + result.len() as u64, + )?; + #[cfg(not(any(unix, windows)))] + let read = { + let _ = (file, capacity); + return Err(invalid("Native checkout reads are unavailable")); + }; + if read == 0 { + return Ok(result); + } + result.extend_from_slice(&chunk[..read]); + if result.len() > limit { + return Err(invalid("Checkout record exceeds limit")); + } + } +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +impl Pins { + fn open(common_dir: &Path, worktree: &str, name: &str) -> io::Result { + if std::fs::symlink_metadata(common_dir)? + .file_type() + .is_symlink() + { + return Err(invalid("Checkout common directory is a symlink")); + } + let common = Dir::open_ambient_dir(common_dir, cap_std::ambient_authority())?; + let root = common.open_dir_nofollow("cyber-worktree-activity")?; + let directory = root.open_dir_nofollow(worktree)?; + let creations = common.open_dir_nofollow("cyber-worktrees")?; + let creation = open_file(&creations, &format!("{name}.json"))?; + Ok(Self { + common, + root, + directory, + creations, + creation, + }) + } + pub(super) fn capture( + managed: &Managed, + owner: &str, + file: &File, + ) -> io::Result<(CheckoutBinding, Self)> { + regular(file)?; + let pins = Self::open(&managed.common_dir, &managed.id, &managed.name)?; + let raw = bytes(&pins.creation)?; + let observed: Managed = serde_json::from_slice(&raw).map_err(invalid)?; + if observed != *managed || !observed.ready { + return Err(invalid("Snapshot checkout creation changed")); + } + let binding = CheckoutBinding { + version: 1, + owner_id: owner.into(), + worktree_id: managed.id.clone(), + name: managed.name.clone(), + common_dir: managed.common_dir.clone(), + common_identity: directory_identity(&pins.common)?, + root_identity: directory_identity(&pins.root)?, + directory_identity: directory_identity(&pins.directory)?, + file_identity: identity(file)?, + creation_directory_identity: directory_identity(&pins.creations)?, + creation_file_identity: identity(&pins.creation)?, + creation_digest: digest(&raw), + }; + Ok((binding, pins)) + } + fn matches(&self, binding: &CheckoutBinding) -> io::Result<()> { + if directory_identity(&self.common)? != binding.common_identity + || directory_identity(&self.root)? != binding.root_identity + || directory_identity(&self.directory)? != binding.directory_identity + || directory_identity(&self.creations)? != binding.creation_directory_identity + || identity(&self.creation)? != binding.creation_file_identity + { + return Err(invalid( + "Snapshot checkout namespace or creation identity changed", + )); + } + Ok(()) + } + pub(super) fn verify(&self, file: &File, record: &UseRecord) -> io::Result<()> { + let binding = record + .snapshot_binding + .as_ref() + .ok_or_else(|| invalid("Missing checkout snapshot binding"))?; + super::safe_id(&binding.owner_id, "op")?; + super::safe_id(&binding.worktree_id, "wt")?; + super::super::Name::parse(&binding.name).map_err(invalid)?; + if binding.version != 1 + || binding.owner_id != record.owner_id + || binding.worktree_id != record.worktree_id + { + return Err(invalid("Snapshot checkout owner changed")); + } + regular(file)?; + if identity(file)? != binding.file_identity { + return Err(invalid("Snapshot checkout descriptor changed")); + } + self.matches(binding)?; + let current = Self::open(&binding.common_dir, &binding.worktree_id, &binding.name)?; + current.matches(binding)?; + if digest(&bytes(¤t.creation)?) != binding.creation_digest { + return Err(invalid("Snapshot checkout creation contents changed")); + } + let observed_file = open_file(¤t.directory, &format!("{}.lock", binding.owner_id))?; + let raw = contents(file, 4096)?; + let observed: UseRecord = serde_json::from_slice(&raw).map_err(invalid)?; + let exact: serde_json::Value = serde_json::from_slice(&raw).map_err(invalid)?; + if identity(&observed_file)? != binding.file_identity + || observed != *record + || exact != serde_json::to_value(record).map_err(invalid)? + { + return Err(invalid("Snapshot checkout route or record changed")); + } + Ok(()) + } +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Queries only live owned handles into fixed-size Windows structures. +mod windows { + use super::*; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_REPARSE_POINT, FILE_ID_INFO, FileIdInfo, + GetFileInformationByHandle, GetFileInformationByHandleEx, + }; + fn info(file: &File) -> io::Result { + let mut value = BY_HANDLE_FILE_INFORMATION::default(); + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut value) } == 0 { + return Err(io::Error::last_os_error()); + } + if value.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return Err(invalid("Snapshot checkout reparse point")); + } + Ok(value) + } + pub(super) fn regular(file: &File) -> io::Result<()> { + if info(file)?.nNumberOfLinks != 1 { + return Err(invalid("Snapshot checkout record has aliases")); + } + Ok(()) + } + pub(super) fn identity(file: &File) -> io::Result { + let info = info(file)?; + let mut id = FILE_ID_INFO::default(); + if unsafe { + GetFileInformationByHandleEx( + file.as_raw_handle(), + FileIdInfo, + (&mut id as *mut FILE_ID_INFO).cast(), + std::mem::size_of::() as u32, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + Ok(format!( + "{}:{:032x}:{}:{}:{}", + id.VolumeSerialNumber, + u128::from_le_bytes(id.FileId.Identifier), + info.ftCreationTime.dwHighDateTime, + info.ftCreationTime.dwLowDateTime, + info.dwFileAttributes + )) + } +} diff --git a/crates/cyber-core/tests/worktree_repository/activity.rs b/crates/cyber-core/tests/worktree_repository/activity.rs index a69c03263..a7df2d275 100644 --- a/crates/cyber-core/tests/worktree_repository/activity.rs +++ b/crates/cyber-core/tests/worktree_repository/activity.rs @@ -1,6 +1,108 @@ use super::*; use cyber_core::worktrees::CheckoutActivity; +#[cfg(unix)] +#[test] +fn snapshot_native_checkout_proof_refuses_aliases_and_an_identical_replacement_record() { + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("snapshot-proof").unwrap(), + &Settings::default(), + ) + .unwrap(); + let worker = + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")).unwrap(); + worker.verify_snapshot().unwrap(); + let binding = worker.snapshot_binding().unwrap().clone(); + let directory = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + let path = directory.join("op_worker.lock"); + let raw = std::fs::read(&path).unwrap(); + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path).unwrap().permissions().mode(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode ^ 0o200)).unwrap(); + assert!(worker.verify_snapshot().is_err()); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).unwrap(); + worker.verify_snapshot().unwrap(); + let record: serde_json::Value = serde_json::from_slice(&raw).unwrap(); + let mut changed = record.clone(); + changed["unrecognized_evidence"] = serde_json::json!("retained"); + std::fs::write(&path, serde_json::to_vec(&changed).unwrap()).unwrap(); + assert!(worker.verify_snapshot().is_err()); + std::fs::write(&path, &raw).unwrap(); + worker.verify_snapshot().unwrap(); + assert_eq!( + record["snapshot_binding"], + serde_json::to_value(binding).unwrap() + ); + let alias = directory.join("alias"); + std::fs::hard_link(&path, &alias).unwrap(); + assert!(worker.verify_snapshot().is_err()); + std::fs::remove_file(alias).unwrap(); + worker.verify_snapshot().unwrap(); + let replacement = directory.join("replacement"); + std::fs::write(&replacement, &raw).unwrap(); + std::fs::rename(replacement, &path).unwrap(); + assert!(worker.verify_snapshot().is_err()); + assert!(worker.settle_retained().is_err()); + assert_eq!(std::fs::read(path).unwrap(), raw); +} + +#[cfg(unix)] +#[test] +fn snapshot_native_checkout_proof_retains_creation_and_namespace_pins() { + for cut in ["creation_contents", "creation_replacement", "namespace"] { + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("snapshot-route").unwrap(), + &Settings::default(), + ) + .unwrap(); + let worker = + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")).unwrap(); + let creation = managed + .common_dir + .join("cyber-worktrees/snapshot-route.json"); + let raw = std::fs::read(&creation).unwrap(); + match cut { + "creation_contents" => { + let mut changed = managed.clone(); + changed.ready = false; + std::fs::write(&creation, serde_json::to_vec(&changed).unwrap()).unwrap(); + assert!(worker.verify_snapshot().is_err()); + std::fs::write(&creation, &raw).unwrap(); + worker.verify_snapshot().unwrap(); + continue; + } + "creation_replacement" => { + let replacement = creation.with_extension("replacement"); + std::fs::write(&replacement, &raw).unwrap(); + std::fs::rename(replacement, &creation).unwrap(); + } + _ => { + let directory = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + let record = std::fs::read(directory.join("op_worker.lock")).unwrap(); + std::fs::rename(&directory, directory.with_extension("retained")).unwrap(); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("op_worker.lock"), record).unwrap(); + } + } + assert!(worker.verify_snapshot().is_err(), "{cut}"); + assert!(worker.settle_retained().is_err(), "{cut}"); + } +} + #[test] fn snapshot_worker_owns_a_distinct_checkout_lock_through_retained_settlement() { let fixture = Fixture::new(); @@ -32,6 +134,9 @@ fn snapshot_worker_owns_a_distinct_checkout_lock_through_retained_settlement() { .unwrap_err(); assert!(error.to_string().contains("op_worker"), "{error}"); drop(proof); + assert!( + block_on(repository.claim_snapshot(&fixture.execution, &managed, "op_worker")).is_err() + ); block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)).unwrap(); } diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 34f49e488..e84f72975 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -341,7 +341,8 @@ impl Scope { sequence: i64, events: Vec, intent_digest: Option<&str>, - ) -> Result<(), RuntimeError> { + location: Box, + ) -> Result, RuntimeError> { let runtime = Runtime { inner: self.inner.upgrade().ok_or(RuntimeError::ShuttingDown)?, }; @@ -376,21 +377,33 @@ impl Scope { (state.info.id.clone(), state.last_seq), (self.id.clone(), self.sequence()), ]; - let (stored, ()) = self.store.append_pair_checked(owners, move |tx| { - if !super::admission_authority::bindings_current(tx, &bindings)? { - return Err(cyber_store::StoreError::InvalidAppend( - super::admission_authority::STALE.into(), - )); - } - Ok(([events, vec![acknowledgement]], ())) - })?; + let stop = self.control.stop.clone(); + let (stored, location) = self + .store + .append_pair_checked(owners, move |tx| { + location + .verify() + .map_err(cyber_store::StoreError::InvalidAppend)?; + if stop.is_cancelled() { + return Err(cyber_store::StoreError::InvalidAppend( + "Snapshot consuming acknowledgement was cancelled".into(), + )); + } + if !super::admission_authority::bindings_current(tx, &bindings)? { + return Err(cyber_store::StoreError::InvalidAppend( + super::admission_authority::STALE.into(), + )); + } + Ok(([events, vec![acknowledgement]], location)) + }) + .map_err(consuming_error)?; self.record = settled; self.settled = true; for event in &stored[0] { state.apply(event).map_err(RuntimeError::Corrupt)?; } runtime.inner.publish(&stored[0]); - Ok(()) + Ok(location) } pub fn settle(&mut self) -> Result<(), RuntimeError> { if matches!(self.authority, Authority::Recovery(_)) { @@ -412,7 +425,8 @@ impl Scope { state: &super::SessionState, sequence: i64, receipt: &super::SnapshotRecoveryReceipt, - ) -> Result<(), RuntimeError> { + location: Box, + ) -> Result, RuntimeError> { self.verify_authority()?; let Authority::Recovery(authority) = &self.authority else { return Err(RuntimeError::Conflict( @@ -468,42 +482,52 @@ impl Scope { (original.activity_id.clone(), original_sequence), (self.id.clone(), self.sequence()), ]; - self.store.append_pair_checked(owners, move |tx| { - let (_, mut original_record) = super::snapshot_recovery_record::attest(tx, &original)?; - let (_, resolution_record) = super::snapshot_recovery_record::attest(tx, &origin)?; - let current: i64 = tx.query_row( - "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", - [&original.session_id], - |row| row.get(0), - )?; - let created: String = tx.query_row( - "SELECT id FROM event WHERE aggregate_id=?1 ORDER BY seq LIMIT 1", - [&original.session_id], - |row| row.get(0), - )?; - if stop.is_cancelled() - || !authority.current(tx)? - || current != sequence - || created != original.session_created_id - || original_record.status == "settled" - || resolution_record.status != "pending" - || resolution_record.snapshot_request.as_ref() != Some(&expected_request) - { - return Err(cyber_store::StoreError::InvalidAppend( - "Recovery consuming acknowledgement authority changed".into(), - )); - } - original_record.status = "settled".into(); - original_record.phase = "snapshot_recovery_committed".into(); - original_record.admission_bindings = None; - original_record.snapshot_receipt = Some(receipt); - Ok(([vec![change(&original_record)], vec![resolution_event]], ())) - })?; + let (_, location) = self + .store + .append_pair_checked(owners, move |tx| { + location + .verify() + .map_err(cyber_store::StoreError::InvalidAppend)?; + let (_, mut original_record) = + super::snapshot_recovery_record::attest(tx, &original)?; + let (_, resolution_record) = super::snapshot_recovery_record::attest(tx, &origin)?; + let current: i64 = tx.query_row( + "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", + [&original.session_id], + |row| row.get(0), + )?; + let created: String = tx.query_row( + "SELECT id FROM event WHERE aggregate_id=?1 ORDER BY seq LIMIT 1", + [&original.session_id], + |row| row.get(0), + )?; + if stop.is_cancelled() + || !authority.current(tx)? + || current != sequence + || created != original.session_created_id + || original_record.status == "settled" + || resolution_record.status != "pending" + || resolution_record.snapshot_request.as_ref() != Some(&expected_request) + { + return Err(cyber_store::StoreError::InvalidAppend( + "Recovery consuming acknowledgement authority changed".into(), + )); + } + original_record.status = "settled".into(); + original_record.phase = "snapshot_recovery_committed".into(); + original_record.admission_bindings = None; + original_record.snapshot_receipt = Some(receipt); + Ok(( + [vec![change(&original_record)], vec![resolution_event]], + location, + )) + }) + .map_err(consuming_error)?; self.handle .acknowledge_snapshot_uncertainty(&request.original.activity_id); self.record = settled; self.settled = true; - Ok(()) + Ok(location) } pub async fn run(&self, future: impl std::future::Future) -> T { let Authority::Ordinary(authority) = &self.authority else { @@ -620,6 +644,13 @@ impl Drop for Scope { self.control.done.cancel(); } } +fn consuming_error(error: cyber_store::StoreError) -> RuntimeError { + match error { + cyber_store::StoreError::InvalidAppend(reason) => RuntimeError::Conflict(reason), + error => error.into(), + } +} + fn change(record: &Record) -> NewEvent { NewEvent::new( CHANGED, diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 64ba1756d..8e9846a33 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -20,6 +20,24 @@ pub trait LocationGuard: Send { fn settle_retained(self: Box) -> Result, String> { Err("Location host does not support retained native settlement".into()) } + fn settle_snapshot_retained(self: Box) -> Result, String> { + Err("Exact retained snapshot checkout proof is unsupported".into()) + } +} + +/// Native metadata proof retained by the actual synchronous snapshot worker. +/// Verification and proof disposal must not reenter the primary runtime/Store, +/// launch processes or perform source effects; disposal may run in the writer. +pub trait SnapshotLocationGuard: Send { + fn verify(&self) -> Result<(), String>; +} +struct UnmanagedSnapshotLocation { + _owner: Box, +} +impl SnapshotLocationGuard for UnmanagedSnapshotLocation { + fn verify(&self) -> Result<(), String> { + Ok(()) + } } pub struct LocationLease { @@ -59,6 +77,17 @@ impl LocationLease { |guard| guard.settle_retained(), ) } + + pub fn settle_snapshot_retained(self) -> Result, String> { + if self.worktree_id.is_none() { + return Ok(Box::new(UnmanagedSnapshotLocation { + _owner: self.settle_retained()?, + })); + } + self.guard + .ok_or("Managed checkout proof is missing")? + .settle_snapshot_retained() + } } /// Host-owned managed child reporting/cleanup, retained until the Drain settles it. diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 8e8513e68..3bf95558d 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -246,9 +246,9 @@ pub use host::{ FileDiff, FileReadError, GoalWorktree, HookEffects, Invocation, LifecycleHook, LifecycleVerdict, LocationGuard, LocationLease, ModelResolver, NoSnapshots, NoTools, Provenance, Reconciliation, ResolvedModel, RestoreError, RuleEffect, RunWorktree, - SessionEnvelope, SkillSuggestion, Snapshot, SnapshotRecovery, Snapshots, TeamWorktree, ToolDef, - ToolHost, ToolOutcome, ToolScope, TurnContext, TurnWorktree, WorkflowAgentOutput, - WorkflowAgentRequest, WorkflowToolRequest, + SessionEnvelope, SkillSuggestion, Snapshot, SnapshotLocationGuard, SnapshotRecovery, Snapshots, + TeamWorktree, ToolDef, ToolHost, ToolOutcome, ToolScope, TurnContext, TurnWorktree, + WorkflowAgentOutput, WorkflowAgentRequest, WorkflowToolRequest, }; pub use model::{ AssistantEntry, CallState, CallStatus, Delivery, Entry, InboxRow, InputStatus, LastStep, diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 962221c4c..242f43909 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -96,7 +96,7 @@ pub struct SnapshotWorker { directory: PathBuf, created_id: String, worktree_id: Option, - _location: Box, + _location: Option>, _lifecycle: OwnedRwLockReadGuard<()>, // Last: releasing this lock proves all earlier worker resources were dropped. _lifetime: Option>, @@ -159,6 +159,7 @@ impl SnapshotWork<'_> { } } pub fn authorize(&self) -> Result<(), String> { + self.owner.location()?.verify()?; if let Some(lifetime) = &self.owner._lifetime { lifetime.verify()?; } @@ -245,6 +246,11 @@ impl SnapshotWork<'_> { } impl SnapshotWorker { + fn location(&self) -> Result<&dyn super::SnapshotLocationGuard, String> { + self._location + .as_deref() + .ok_or_else(|| "Snapshot checkout proof is unavailable".into()) + } /// Trusted adapters must perform only synchronous filesystem work here; /// spawning a process or detached task cannot use this acknowledgement proof. pub async fn run( @@ -260,6 +266,10 @@ impl SnapshotWorker { .guard .verify(self.scope.snapshot_origin()) .map_err(RestoreError::Failed)?; + self.location() + .map_err(RestoreError::Failed)? + .verify() + .map_err(RestoreError::Failed)?; Ok(SnapshotCompletion { worker: self, native, @@ -291,9 +301,18 @@ impl SnapshotCompletion { .guard .verify(self.worker.scope.snapshot_origin()) .map_err(RuntimeError::Conflict)?; - self.worker - .scope - .commit_recovery(state, self.worker.sequence, &receipt)?; + let location = self + .worker + ._location + .take() + .ok_or_else(|| RuntimeError::Conflict("Missing snapshot checkout proof".into()))?; + location.verify().map_err(RuntimeError::Conflict)?; + self.worker._location = Some(self.worker.scope.commit_recovery( + state, + self.worker.sequence, + &receipt, + location, + )?); self.native.guard.acknowledge(self.worker.scope.snapshot_origin()).map_err(|error| RuntimeError::Conflict(format!( "Recovery runtime acknowledgement committed; native journal acknowledgement requires reconciliation: {error}" )))?; @@ -313,12 +332,19 @@ impl SnapshotCompletion { .guard .verify(self.worker.scope.snapshot_origin()) .map_err(RuntimeError::Conflict)?; - self.worker.scope.commit_snapshot( + let location = self + .worker + ._location + .take() + .ok_or_else(|| RuntimeError::Conflict("Missing snapshot checkout proof".into()))?; + location.verify().map_err(RuntimeError::Conflict)?; + self.worker._location = Some(self.worker.scope.commit_snapshot( state, self.worker.sequence, events, self.native.guard.intent_digest(), - )?; + location, + )?); self.native .guard .acknowledge(self.worker.scope.snapshot_origin()) @@ -408,7 +434,11 @@ impl Runtime { directory, created_id, worktree_id: state.info.worktree_id.clone(), - _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, + _location: Some( + lease + .settle_snapshot_retained() + .map_err(RuntimeError::Invalid)?, + ), _lifecycle: lifecycle, _lifetime: lifetime, }) @@ -464,7 +494,11 @@ impl Runtime { directory, created_id, worktree_id, - _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, + _location: Some( + lease + .settle_snapshot_retained() + .map_err(RuntimeError::Invalid)?, + ), _lifecycle: lifecycle, _lifetime: lifetime, admission, diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 3a37cc50b..52b28b2f5 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -190,6 +190,189 @@ async fn default_host_refuses_managed_snapshot_claim_without_explicit_support() } struct Acknowledgements(Arc); +struct CheckoutProof { + calls: Arc, + fail_at: Arc, +} + +struct StopCheckoutProof { + calls: Arc, + stop_at: Arc, + stop: tokio_util::sync::CancellationToken, +} +impl super::super::SnapshotLocationGuard for StopCheckoutProof { + fn verify(&self) -> Result<(), String> { + if self.calls.fetch_add(1, Ordering::SeqCst) + 1 == self.stop_at.load(Ordering::SeqCst) { + self.stop.cancel(); // Model Stop arriving during the writer's final native check. + } + Ok(()) + } +} + +#[tokio::test] +async fn stop_during_writer_checkout_verification_refuses_consuming_acknowledgement() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let mut owner = worker(&runtime, &session).await; + let calls = Arc::new(AtomicUsize::new(0)); + let stop_at = Arc::new(AtomicUsize::new(usize::MAX)); + owner._location = Some(Box::new(StopCheckoutProof { + calls: calls.clone(), + stop_at: stop_at.clone(), + stop: owner.scope.control.stop.clone(), + })); + let binding = binding(temp.path().canonicalize().unwrap()); + let acknowledgements = Arc::new(AtomicUsize::new(0)); + let acknowledged = acknowledgements.clone(); + let completed = owner + .run(move |work| { + let origin = work.begin(binding).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: acknowledged, + }), + )) + }) + .await + .unwrap(); + stop_at.store(calls.load(Ordering::SeqCst) + 2, Ordering::SeqCst); + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + let before = state.last_seq; + let result = completed.commit( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "must not commit after Stop".into(), + usage: None, + cost: None, + model: None, + }, + )], + ); + assert!( + matches!(result, Err(RuntimeError::Conflict(ref reason)) if reason.contains("cancelled")) + ); + assert_eq!(state.last_seq, before); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); + drop(state); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + runtime.shutdown().await; +} +impl super::super::SnapshotLocationGuard for CheckoutProof { + fn verify(&self) -> Result<(), String> { + let call = self.calls.fetch_add(1, Ordering::SeqCst) + 1; + if call >= self.fail_at.load(Ordering::SeqCst) { + return Err("checkout native route changed".into()); + } + Ok(()) + } +} + +#[tokio::test] +async fn changed_checkout_proof_refuses_the_source_callback() { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let calls = Arc::new(AtomicUsize::new(0)); + let fail_at = Arc::new(AtomicUsize::new(usize::MAX)); + let mut owner = worker(&runtime, &session).await; + owner._location = Some(Box::new(CheckoutProof { + calls: calls.clone(), + fail_at: fail_at.clone(), + })); + let source_calls = Arc::new(AtomicUsize::new(0)); + let source = source_calls.clone(); + let binding = binding(temp.path().canonicalize().unwrap()); + assert!( + owner + .run(move |work| { + work.begin(binding).unwrap(); + fail_at.store(calls.load(Ordering::SeqCst) + 1, Ordering::SeqCst); + assert!( + work.effect(|| { + source.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .is_err() + ); + Err(RestoreError::Failed( + "changed checkout before source effect".into(), + )) + }) + .await + .is_err() + ); + assert_eq!(source_calls.load(Ordering::SeqCst), 0); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + runtime.shutdown().await; +} + +#[tokio::test] +async fn consuming_writer_verifies_retained_checkout_proof_and_rolls_back_both_owners_on_change() { + for changed in [false, true] { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let calls = Arc::new(AtomicUsize::new(0)); + let fail_at = Arc::new(AtomicUsize::new(usize::MAX)); + let mut owner = worker(&runtime, &session).await; + owner._location = Some(Box::new(CheckoutProof { + calls: calls.clone(), + fail_at: fail_at.clone(), + })); + let binding = binding(temp.path().canonicalize().unwrap()); + let acknowledgements = Arc::new(AtomicUsize::new(0)); + let acknowledged = acknowledgements.clone(); + let completed = owner + .run(move |work| { + let origin = work.begin(binding).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: acknowledged, + }), + )) + }) + .await + .unwrap(); + let before_checks = calls.load(Ordering::SeqCst); + if changed { + fail_at.store(before_checks + 2, Ordering::SeqCst); + } + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + let before = state.last_seq; + let result = completed.commit( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "checkout consuming batch".into(), + usage: None, + cost: None, + model: None, + }, + )], + ); + assert_eq!(result.is_err(), changed); + if changed { + assert!(matches!(result, Err(RuntimeError::Conflict(_)))); + } + assert_eq!(calls.load(Ordering::SeqCst), before_checks + 2); + assert_eq!(state.last_seq, before + i64::from(!changed)); + assert_eq!( + acknowledgements.load(Ordering::SeqCst), + usize::from(!changed) + ); + drop(state); + assert_eq!( + latest(&runtime, &session)["status"], + if changed { "unknown" } else { "settled" } + ); + runtime.shutdown().await; + } +} + struct Lifetimes { inner: Acknowledgements, permits: Arc, diff --git a/crates/cyber-store/migrations/20261106020000_snapshot_checkout_proof.sql b/crates/cyber-store/migrations/20261106020000_snapshot_checkout_proof.sql new file mode 100644 index 000000000..e768a0e40 --- /dev/null +++ b/crates/cyber-store/migrations/20261106020000_snapshot_checkout_proof.sql @@ -0,0 +1,3 @@ +-- Compatibility marker: older writers must not bypass native checkout proof +-- verification in snapshot callbacks and consuming acknowledgements. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index fb2e16e1f..feaae4ae9 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -181,6 +181,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261106010000_snapshot_source_admission", sql: include_str!("../migrations/20261106010000_snapshot_source_admission.sql"), }, + Migration { + id: "20261106020000_snapshot_checkout_proof", + sql: include_str!("../migrations/20261106020000_snapshot_checkout_proof.sql"), + }, ]; pub(crate) enum Outcome { @@ -352,7 +356,13 @@ mod tests { let Outcome::TooNew(unknown) = migrate(&mut reopened, legacy).unwrap() else { panic!("previous worker catalog admitted source-admission storage"); }; - assert_eq!(unknown, ["20261106010000_snapshot_source_admission"]); + assert_eq!( + unknown, + MIGRATIONS[at..] + .iter() + .map(|migration| migration.id) + .collect::>() + ); let retained: String = reopened .query_row("SELECT data FROM event WHERE id='evt_source'", [], |row| { row.get(0) @@ -361,6 +371,37 @@ mod tests { assert_eq!(retained, evidence); } + #[test] + fn checkout_proof_marker_rejects_previous_writers_without_changing_native_evidence() { + let directory = tempfile::tempdir().unwrap(); + let database = directory.path().join("checkout.db"); + let claim = directory.path().join("op_worker.lock"); + let evidence = + br#"{"owner_id":"op_worker","settled":false,"snapshot_binding":{"version":1}}"#; + std::fs::write(&claim, evidence).unwrap(); + let at = MIGRATIONS + .iter() + .position(|migration| migration.id == "20261106020000_snapshot_checkout_proof") + .unwrap(); + let previous = &MIGRATIONS[..at]; + let mut db = Connection::open(&database).unwrap(); + assert!(matches!( + migrate(&mut db, previous).unwrap(), + Outcome::Current + )); + assert!(matches!( + migrate(&mut db, MIGRATIONS).unwrap(), + Outcome::Current + )); + drop(db); + let mut db = Connection::open(database).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut db, previous).unwrap() else { + panic!("previous writers bypassed checkout proof admission"); + }; + assert_eq!(unknown, ["20261106020000_snapshot_checkout_proof"]); + assert_eq!(std::fs::read(claim).unwrap(), evidence); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index 51bc79468..e2b029ebd 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -1212,6 +1212,35 @@ impl cyber_server::runtime::LocationGuard for ActiveLocation { let proof = lease.settle_retained()?; Ok(Box::new((proof, activity))) } + fn settle_snapshot_retained( + self: Box, + ) -> Result, String> { + let Self { lease, activity } = *self; + Ok(Box::new(ActiveSnapshotLocation { + proof: lease.settle_snapshot_retained()?, + _activity: activity, + })) + } +} + +struct ActiveSnapshotLocation { + proof: Box, + _activity: crate::lsp::Activity, +} +impl cyber_server::runtime::SnapshotLocationGuard for ActiveSnapshotLocation { + fn verify(&self) -> Result<(), String> { + self.proof.verify() + } +} + +struct SnapshotWorktreeProof(Vec); +impl cyber_server::runtime::SnapshotLocationGuard for SnapshotWorktreeProof { + fn verify(&self) -> Result<(), String> { + for lease in &self.0 { + lease.verify_snapshot().map_err(|error| error.to_string())?; + } + Ok(()) + } } impl cyber_server::runtime::LocationGuard for WorktreeLease { @@ -1230,6 +1259,21 @@ impl cyber_server::runtime::LocationGuard for WorktreeLease { .collect::, _>>()?; Ok(Box::new(leases)) } + fn settle_snapshot_retained( + self: Box, + ) -> Result, String> { + for lease in &self.0 { + lease.verify_snapshot().map_err(|error| error.to_string())?; + } + let leases = self + .0 + .into_iter() + .map(|lease| lease.settle_retained().map_err(|error| error.to_string())) + .collect::, _>>()?; + let proof = SnapshotWorktreeProof(leases); + cyber_server::runtime::SnapshotLocationGuard::verify(&proof)?; + Ok(Box::new(proof)) + } } impl GitExecution for GitPort<'_> { diff --git a/crates/cyber-tools/tests/worktree_setup.rs b/crates/cyber-tools/tests/worktree_setup.rs index 02eb200c8..e2f0c77ef 100644 --- a/crates/cyber-tools/tests/worktree_setup.rs +++ b/crates/cyber-tools/tests/worktree_setup.rs @@ -1912,6 +1912,85 @@ async fn process_launch_failure_remains_uncertain_and_cannot_authorize_retry() { assert!(!managed.path.join("target.txt").exists()); } +#[tokio::test] +async fn managed_git_legacy_rewind_preserves_an_independent_native_checkout_proof() { + use cyber_server::runtime::{CreateSession, RevertTarget, ToolHost}; + let (fixture, _, managed) = owned().await; + #[cfg(windows)] + fixture.set_config(json!({"sandbox":{"policy":"full-access"}})); + let config = fixture.config.clone(); + let snapshots = cyber_snapshot::GitSnapshots::new( + fixture.dir.path().join("data"), + Arc::new(move |_| config.lock().unwrap().clone()), + ); + let flow = support::flow::Flow::with( + fixture, + vec![support::flow::text("before"), support::flow::text("after")], + false, + snapshots, + ); + let info = flow + .runtime + .create_session(CreateSession { + directory: managed.path.display().to_string(), + model: "test/main".into(), + mode: Some("bypass".into()), + ..Default::default() + }) + .await + .unwrap(); + let first = flow.prompt(&info.id, "before external edit").await; + flow.settle(&info.id).await; + std::fs::write(managed.path.join("tracked.txt"), "after").unwrap(); + flow.prompt(&info.id, "after external edit").await; + flow.settle(&info.id).await; + // Git still uses the legacy restore path. Exercise its compatibility while + // an independently acquired snapshot checkout proof remains retained. + let claim = flow + .f + .host + .claim_snapshot_location(&info, "op_retained", CancellationToken::new()) + .await + .unwrap(); + let proof = claim.settle_snapshot_retained().unwrap(); + proof.verify().unwrap(); + flow.runtime + .revert_stage(&info.id, &first, RevertTarget::Code) + .await + .unwrap(); + flow.runtime.revert_commit(&info.id).await.unwrap(); + proof.verify().unwrap(); + drop(proof); + assert_eq!( + std::fs::read_to_string(managed.path.join("tracked.txt")).unwrap(), + "base" + ); + let directory = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + let claims = std::fs::read_dir(directory) + .unwrap() + .filter_map(|entry| { + let path = entry.unwrap().path(); + path.file_name() + .unwrap() + .to_string_lossy() + .starts_with("op_") + .then_some(path) + }) + .collect::>(); + assert!(!claims.is_empty()); + for path in claims { + let record: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + assert_eq!(record["snapshot_binding"]["worktree_id"], managed.id); + assert_eq!(record["snapshot_binding"]["owner_id"], record["owner_id"]); + assert_eq!(record["settled"], true); + } + flow.runtime.shutdown().await; +} + #[tokio::test] async fn snapshot_location_claim_is_independent_of_a_live_session_and_retained_until_disposal() { use cyber_core::worktrees::CheckoutActivity; @@ -1944,7 +2023,20 @@ async fn snapshot_location_claim_is_independent_of_a_live_session_and_retained_u .unwrap(); assert_eq!(worker.worktree_id.as_deref(), Some(managed.id.as_str())); actor.settle().unwrap(); - let proof = worker.settle_retained().unwrap(); + let proof = worker.settle_snapshot_retained().unwrap(); + proof.verify().unwrap(); + #[cfg(unix)] + { + let directory = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + let alias = directory.join("snapshot-alias"); + std::fs::hard_link(directory.join("op_snapshot.lock"), &alias).unwrap(); + assert!(proof.verify().is_err()); + std::fs::remove_file(alias).unwrap(); + proof.verify().unwrap(); + } let error = repository .remove(&Git, &CheckoutActivity, &managed, false) .await diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 7849e63f2..98cd87412 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -164,3 +164,10 @@ from Session IDs to independent native activity IDs through a dedicated host por CLI/App retain those locks through settlement; unsupported managed hosts refuse. Unknown checkout claims remain protected. Exact interrupted-claim proof and reconciliation are still required before accepting managed resource recovery. + + +Acquired snapshot checkout claims now carry pinned native route/creation evidence. +The actual worker and consuming Store writer verify and retain that proof; aliases, +replacement and changed records refuse. Binding publication before interrupted +acquisition and recovery-owned native acknowledgement remain required. Git rewind +still follows its legacy path; compatibility tests do not establish owned Git recovery. diff --git a/docs/rewind.md b/docs/rewind.md index db5a4e3fd..05fb73df0 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -296,3 +296,10 @@ explicit support refuse managed snapshot admission. A crash while acquiring a cl can still leave unknown checkout evidence: preparation cancellation does not clear that evidence or authorize checkout removal. Exact native claim reconciliation is still required for that acquisition cut; retain the evidence for recovery. + + +While a managed snapshot is running, the worker verifies its pinned checkout claim +and worktree creation metadata before source changes and consuming commits. Aliased +or replaced claim files, changed creation contents and replaced activity directories +refuse. These checks preserve evidence; they do not grant permission to clear an +unknown checkout claim after interrupted acquisition. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 109ea4693..fe86ca477 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1227,3 +1227,88 @@ ports and refusal paths. No critical/major findings remain for independent acqui immutable interrupted-claim proof and full managed recovery acceptance remain open. This continuation remains local and does not authorize unknown claim settlement, complete M2–M8 or make the full recovery increment merge-ready. + + +### Exact proof for acquired native checkout claims (2026-11-06, local) + +New snapshot checkout leases retain native common/activity/creation directory pins, +the creation descriptor and the locked claim. A bounded versioned native binding +records exact identities, owner/worktree, creation digest and Unix owner/mode (or +Windows full file/volume/creation identity and attributes). Verification opens fresh +no-follow child routes, refuses aliases/replacements, compares creation bytes and +reads the exact claim using positional reads on its original locked descriptor. +Unknown record fields refuse verification. Snapshot activity claims use exclusive +creation and are never reset, including after settlement; generic Session behavior +is unchanged. Legacy/unbound claims do not acquire new proof. + +The host retains a dedicated `SnapshotLocationGuard`; managed guards must implement +it explicitly. Normal workers verify it before authorization, after native completion +and at consuming commit. The same proof moves into the Store writer, is verified +there, and is returned only after the observed paired commit. It therefore remains +held through actual writer completion/reply. A failed native verification rolls back +both aggregates and prevents journal acknowledgement. The callback performs bounded +native metadata work and cannot call the primary runtime/Store or launch processes. +Unmanaged owners retain their existing behavior without inventing managed proof. + +Native regressions exercise claim mode/unknown-field changes, hard links, identical +file replacement, altered/replaced creation records and replaced activity namespaces. +Runtime regression counters verify that changed proof prevents the source callback +and that writer-local rejection rolls back both owners, alongside a successful +consuming commit. Production host coverage uses the exact native guard and refuses +an introduced hard link without releasing retained ownership. + +An initial new integration assertion expected legacy Git rewind to issue an owned +snapshot claim. It found no such claim: Git still uses the existing restore path. +The corrected test verifies that legacy Git restoration preserves an explicitly +retained independent native checkout proof, rather than claiming owned Git recovery. +Git journaling/owned restoration remains required and is not accepted by that test. + +This continuation verifies acquired resources; it does not yet publish the complete +checkout binding through the immutable snapshot lifetime before stable acquisition, +reacquire interrupted unknown claims, or consume native checkout acknowledgements +from recovery receipts. The incomplete binding-publication cut retains evidence and +cannot be force-cleared. Full M2–M8, native platform/cut coverage, reclamation and +client acceptance remain open. No parent task or issue is complete. + + +Local verification: 420 server unit tests passed with one ignored subprocess fixture, +plus 22 affected integrations (4 idle cancellation, 13 Location activity and 5 rewind +preview). The reviewed core worktree repository suite passed all 70 cases with its +subprocess child listed ignored; all five selected activity unit regressions passed. +All 39 production worktree host cases passed after correcting the Git compatibility +assertion described above. All 16 real CLI branch/recovery cases passed, including +real early/partial kills and request reconciliation. These broad checks were followed +by focused core/host verification and all-target Clippy after the final core review +changes. Store tests passed all 60 cases after adding the checkout-proof compatibility +marker; the previous catalog refuses mutation and native bytes remain unchanged. +All-target core/server/tools/App/CLI Clippy passed with warnings denied. Formatting, +diff, generated SDK consistency and strict pinned OpenSpec (117 items) passed; spec +lint reports 0 errors/25 warnings. No HTTP payload changed, and SDK execution tests +were not rerun for these Rust/native-only changes. Full-workspace tests, native +Linux/Windows execution and live TUI/App recovery acceptance were not run. Native +sandbox deprecation/Darwin linker warnings were observed without attributing an +origin to main. Remote main remains `db9e171e`. + +The code-review skill was applied to all changed core/native, runtime/writer, host, +Store migration, test and documentation files. The metadata port keeps the existing +runtime/native-adapter direction. Review checks descriptor and namespace retention, +creation/record bounds, exact IDs and metadata, no-follow/alias handling, positional +reads under Windows locks, one-shot exclusive publication, writer-local proof +failure rollback, returned proof lifetime, and old-writer compatibility. Verification +and proof disposal may occur in the writer and must not reenter the primary Store; +the provided native adapters only query/release native resources there. Review added +exclusive claim creation, strict unknown-field comparison and the compatibility gate. +No critical/major findings remain for acquired-claim verification. Unknown acquisition +recovery, complete immutable lifetime binding and owned Git journaling remain explicit +acceptance gaps; this local increment does not complete either issue or the full +recovery delivery. + + +Final review also adds a writer-local Stop check after native checkout verification; +a cancelled ordinary restore cannot consume its acknowledgement merely because its +caller-side check ran before queueing. The targeted regression injects Stop during +the final native check and verifies unchanged Session state, unknown native origin +and no journal acknowledgement. Checked owner refusals return `Conflict` while +schema, corruption and storage failures retain their distinct error paths. The +final snapshot-worker suite (34 tests) and server Clippy passed after this change. +The checkout-proof migration has a separate passing Store all-target Clippy check. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 3da5f9557..4801cb65c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -301,3 +301,34 @@ Session. Exact persisted checkout file/directory identity and creation/lifetime binding, consuming proof through Store settlement, reconciliation and real kill cuts across multi-checkout acquisition remain required before the complete resource recovery path is accepted. Older Session-owned unknown leases remain independent. + + +### Native proof retained during acquired checkout work + +Each new snapshot checkout claim now records a bounded versioned binding for the +claim file, common/activity directories and managed creation record. The core lease +pins the actual directories/creation descriptor alongside its locked claim and +verifies a fresh no-follow route against those identities, exact owner/worktree, +creation digest and retained record. Unix identity includes device/inode/birth and +owner/mode; Windows uses full volume/file identity, creation time and attributes, +with reparse and hard-link refusal. Positional reads use the original locked claim +handle and do not reopen its locked byte range for reading. + +The production adapter retains a dedicated metadata verification port, including +Location activity. The worker checks it at authorization, completion and immediately +before consuming settlement. The Store writer verifies that same native proof and +returns it only after the paired commit/reply, so failure rolls back both owners +and suppresses journal acknowledgement. Verification cannot call the primary +runtime/Store or launch Git/processes. Unmanaged ownership keeps its prior retention +semantics without claiming managed proof. + +This is proof of an already acquired claim. It does not yet bind the checkout +receipt to the immutable snapshot lifetime for interrupted-acquisition recovery, +reacquire an unknown claim or publish new acknowledgement authority. The cut before complete native binding publication remains open. Persisting the complete +resource binding before stable publication and consuming it with durable recovery +receipts are still required. No old Session or operation claim is reclassified. + + +The checkout-proof compatibility migration prevents earlier writer catalogs from +opening this Store for mutation and bypassing these native verification steps. +It performs no native evidence repair or deletion. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index b927117cf..e4e01a694 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -99,6 +99,13 @@ - **AND** settling that snapshot claim SHALL NOT acknowledge unrelated Session activity; an interrupted acquisition SHALL retain unknown evidence without a generic clearing path - **AND** a managed host without explicit snapshot ownership support SHALL refuse instead of reusing a Session claim +#### Scenario: Retained native checkout routes change +- **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition +- **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents +- **AND** aliases, replaced native objects and contradictory records SHALL refuse without acknowledging foreign resources +- **AND** the consuming Store writer SHALL verify and retain the exact checkout proof through its actual commit and observed reply +- **AND** previous writer catalogs lacking this proof contract SHALL refuse Store mutation while preserving native evidence + #### Scenario: Lifetime proof precedes reservation - **WHEN** a new file-copy worker reserves runtime ownership before publishing its restore origin - **THEN** an immutable worker-specific native lifetime lock already binds its activity, Session creation, checkout, Location and private namespace, and remains retained through actual resource teardown diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index b88d64e87..b7ba41884 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -189,3 +189,12 @@ unknown claim retention and preservation of unrelated Session evidence. This is acquisition ownership, not native reconciliation authority: immutable persisted checkout route/creation/lifetime binding and interrupted-claim acknowledgement remain incomplete, so M2–M8 and parent tasks remain open. + + +Acquired checkout proof continuation: snapshot claims retain native common/activity/ +creation directory pins and record exact file/creation identity. Runtime source +callbacks and consuming commits verify the dedicated native proof, including inside +the Store writer through commit/reply. Changed routes/records refuse without source +callbacks or acknowledgement. This does not finish immutable lifetime/resource +publication or unknown-claim reacquisition across early cuts. Those and complete +platform/client acceptance keep M2–M8 and parent tasks open. From 6f65d3500399be4a8459263bd35c31c0a2f3007c Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Fri, 6 Nov 2026 23:41:09 -0400 Subject: [PATCH 15/87] Publish initialized checkout claims with exact lifetime context --- crates/cyber-app/src/host.rs | 11 + crates/cyber-core/src/worktrees.rs | 2 +- crates/cyber-core/src/worktrees/activity.rs | 76 ++-- .../src/worktrees/activity/snapshot.rs | 94 ++++- .../activity/snapshot/publication.rs | 325 ++++++++++++++++++ .../tests/worktree_repository/activity.rs | 52 +++ crates/cyber-server/src/runtime/host.rs | 31 ++ crates/cyber-server/src/runtime/location.rs | 10 +- .../src/runtime/snapshot_lifetime.rs | 94 +++-- .../cyber-server/src/runtime/snapshot_work.rs | 71 +++- .../src/runtime/snapshot_work_tests.rs | 112 ++++++ ...61106030000_snapshot_checkout_lifetime.sql | 3 + crates/cyber-store/src/migrate.rs | 45 ++- crates/cyber-tools/src/host.rs | 10 + crates/cyber-tools/src/worktrees.rs | 71 ++++ crates/cyber-tools/tests/worktree_setup.rs | 84 +++++ docs/goal-delivery.md | 8 + docs/rewind.md | 9 + .../implementation.md | 77 +++++ .../managed-recovery.md | 36 ++ .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 12 + 22 files changed, 1156 insertions(+), 83 deletions(-) create mode 100644 crates/cyber-core/src/worktrees/activity/snapshot/publication.rs create mode 100644 crates/cyber-store/migrations/20261106030000_snapshot_checkout_lifetime.sql diff --git a/crates/cyber-app/src/host.rs b/crates/cyber-app/src/host.rs index 48e5299a4..983713f8b 100644 --- a/crates/cyber-app/src/host.rs +++ b/crates/cyber-app/src/host.rs @@ -279,6 +279,17 @@ impl ToolHost for AppHost { self.builtin.claim_snapshot_location(info, activity, cancel) } + fn claim_snapshot_location_bound<'a>( + &'a self, + info: &'a cyber_server::runtime::SessionInfo, + activity: &'a str, + lifetime: Option<&'a cyber_server::runtime::SnapshotLifetimeBinding>, + cancel: CancellationToken, + ) -> BoxFuture<'a, Result> { + self.builtin + .claim_snapshot_location_bound(info, activity, lifetime, cancel) + } + fn prepare_child_continuation<'a>( &'a self, parent: &'a cyber_server::runtime::SessionInfo, diff --git a/crates/cyber-core/src/worktrees.rs b/crates/cyber-core/src/worktrees.rs index e627876b2..f85460f8a 100644 --- a/crates/cyber-core/src/worktrees.rs +++ b/crates/cyber-core/src/worktrees.rs @@ -8,7 +8,7 @@ mod location; mod removal; pub(crate) mod repository; mod setup; -pub use activity::{CheckoutActivity, CheckoutBinding, CheckoutLease}; +pub use activity::{CheckoutActivity, CheckoutBinding, CheckoutLease, CheckoutOwner}; pub use changes::{ChangedFile, Changes}; pub use includes::IncludedFile; pub use removal::{RemovalActivity, RemovalPhase, RemovalRecord}; diff --git a/crates/cyber-core/src/worktrees/activity.rs b/crates/cyber-core/src/worktrees/activity.rs index 7d1dfa4ee..050ece170 100644 --- a/crates/cyber-core/src/worktrees/activity.rs +++ b/crates/cyber-core/src/worktrees/activity.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use super::{GitExecution, Managed, Name, RemovalActivity, Repository, RepositoryLock}; mod snapshot; -pub use snapshot::CheckoutBinding; +pub use snapshot::{CheckoutBinding, CheckoutOwner}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct UseRecord { @@ -193,24 +193,28 @@ impl Repository { activity: &str, ) -> io::Result { safe_id(activity, "op")?; - let mut lease = self.claim_owner(execution, managed, activity).await?; - let (binding, pins) = snapshot::Pins::capture(managed, activity, &lease.file)?; - lease.record.snapshot_binding = Some(binding); - lease.snapshot_pins = Some(pins); - write_record(&mut lease.file, &lease.record)?; - lease.verify_snapshot()?; - Ok(lease) + let _lock = self.claim_ready(execution, managed).await?; + snapshot::publish(managed, activity, None) } - async fn claim_owner( + pub async fn claim_snapshot_bound( &self, execution: &dyn GitExecution, managed: &Managed, - session: &str, + owner: &CheckoutOwner, ) -> io::Result { - safe_owner_id(session)?; + owner.validate(managed)?; + let _lock = self.claim_ready(execution, managed).await?; + snapshot::publish(managed, &owner.activity_id, Some(owner.clone())) + } + + async fn claim_ready( + &self, + execution: &dyn GitExecution, + managed: &Managed, + ) -> io::Result { safe_id(&managed.id, "wt")?; - let _lock = RepositoryLock::try_acquire(&self.common_dir)?.ok_or_else(|| { + let lock = RepositoryLock::try_acquire(&self.common_dir)?.ok_or_else(|| { io::Error::new(io::ErrorKind::WouldBlock, "Worktree repository is busy") })?; if managed.common_dir != self.common_dir { @@ -230,19 +234,20 @@ impl Repository { if stored != *managed { return Err(invalid("Worktree creation identity changed")); } + Ok(lock) + } + + async fn claim_owner( + &self, + execution: &dyn GitExecution, + managed: &Managed, + session: &str, + ) -> io::Result { + safe_owner_id(session)?; + let _lock = self.claim_ready(execution, managed).await?; let path = use_directory(managed, true)?.join(format!("{session}.lock")); let existed = path.try_exists()?; - let file = if crate::ids::has_prefix(session, "op") { - // An activity is a one-shot owner. Preserve old/future evidence even - // after settlement; no new operation may reset this claim. - OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(&path)? - } else { - open_record(&path, true)? - }; + let file = open_record(&path, true)?; if !try_lock(&file)? { return Err(io::Error::other(format!("Worktree in use by {session}"))); } @@ -347,11 +352,24 @@ fn try_lock(file: &File) -> io::Result { fn read_record(file: &mut File) -> io::Result { file.rewind()?; let mut bytes = Vec::new(); - file.take(4097).read_to_end(&mut bytes)?; - if bytes.len() > 4096 { - return Err(invalid("Activity record exceeds 4 KiB")); + file.take(16 * 1024 + 1).read_to_end(&mut bytes)?; + let record = serde_json::from_slice(&bytes).map_err(invalid)?; + if bytes.len() > record_limit(&record) { + return Err(invalid("Activity record exceeds limit")); + } + Ok(record) +} + +fn record_limit(record: &UseRecord) -> usize { + if record + .snapshot_binding + .as_ref() + .is_some_and(|binding| binding.version == 2) + { + 16 * 1024 + } else { + 4096 } - serde_json::from_slice(&bytes).map_err(invalid) } fn check_record(record: &UseRecord, managed: &Managed, session: &str) -> io::Result<()> { @@ -363,8 +381,8 @@ fn check_record(record: &UseRecord, managed: &Managed, session: &str) -> io::Res fn write_record(file: &mut File, record: &UseRecord) -> io::Result<()> { let bytes = serde_json::to_vec(record).map_err(invalid)?; - if bytes.len() > 4096 { - return Err(invalid("Activity record exceeds 4 KiB")); + if bytes.len() > record_limit(record) { + return Err(invalid("Activity record exceeds limit")); } file.rewind()?; file.set_len(0)?; diff --git a/crates/cyber-core/src/worktrees/activity/snapshot.rs b/crates/cyber-core/src/worktrees/activity/snapshot.rs index 805067a49..cc73dfa69 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot.rs @@ -1,7 +1,9 @@ //! Exact checkout metadata pins. A binding alone never grants acknowledgement. use super::{Managed, UseRecord, invalid}; +mod publication; use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, OpenOptions}; +pub(super) use publication::publish; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ @@ -25,6 +27,48 @@ pub struct CheckoutBinding { pub creation_directory_identity: String, pub creation_file_identity: String, pub creation_digest: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lifetime: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staging_name: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckoutOwner { + pub activity_id: String, + pub session_id: String, + pub session_created_id: String, + pub directory: PathBuf, + pub lifetime_digest: String, +} +impl CheckoutOwner { + fn validate_fields(&self) -> io::Result<()> { + super::safe_id(&self.activity_id, "op")?; + super::safe_id(&self.session_id, "ses")?; + if self.session_created_id.is_empty() + || self.session_created_id.len() > 256 + || !self.directory.is_absolute() + || self.directory.to_str().is_none() + || self.lifetime_digest.len() != 64 + || !self + .lifetime_digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(invalid("Invalid snapshot checkout lifetime context")); + } + Ok(()) + } + pub(super) fn validate(&self, managed: &Managed) -> io::Result<()> { + self.validate_fields()?; + if self.directory.canonicalize()? != self.directory + || !self.directory.starts_with(&managed.path) + { + return Err(invalid("Snapshot checkout lifetime Location changed")); + } + Ok(()) + } } pub(super) struct Pins { @@ -152,13 +196,13 @@ impl Pins { }) } pub(super) fn capture( + &self, managed: &Managed, owner: &str, file: &File, - ) -> io::Result<(CheckoutBinding, Self)> { + ) -> io::Result { regular(file)?; - let pins = Self::open(&managed.common_dir, &managed.id, &managed.name)?; - let raw = bytes(&pins.creation)?; + let raw = bytes(&self.creation)?; let observed: Managed = serde_json::from_slice(&raw).map_err(invalid)?; if observed != *managed || !observed.ready { return Err(invalid("Snapshot checkout creation changed")); @@ -169,15 +213,17 @@ impl Pins { worktree_id: managed.id.clone(), name: managed.name.clone(), common_dir: managed.common_dir.clone(), - common_identity: directory_identity(&pins.common)?, - root_identity: directory_identity(&pins.root)?, - directory_identity: directory_identity(&pins.directory)?, + common_identity: directory_identity(&self.common)?, + root_identity: directory_identity(&self.root)?, + directory_identity: directory_identity(&self.directory)?, file_identity: identity(file)?, - creation_directory_identity: directory_identity(&pins.creations)?, - creation_file_identity: identity(&pins.creation)?, + creation_directory_identity: directory_identity(&self.creations)?, + creation_file_identity: identity(&self.creation)?, creation_digest: digest(&raw), + lifetime: None, + staging_name: None, }; - Ok((binding, pins)) + Ok(binding) } fn matches(&self, binding: &CheckoutBinding) -> io::Result<()> { if directory_identity(&self.common)? != binding.common_identity @@ -193,6 +239,9 @@ impl Pins { Ok(()) } pub(super) fn verify(&self, file: &File, record: &UseRecord) -> io::Result<()> { + self.verify_at(file, record, &format!("{}.lock", record.owner_id)) + } + fn verify_at(&self, file: &File, record: &UseRecord, name: &str) -> io::Result<()> { let binding = record .snapshot_binding .as_ref() @@ -200,12 +249,31 @@ impl Pins { super::safe_id(&binding.owner_id, "op")?; super::safe_id(&binding.worktree_id, "wt")?; super::super::Name::parse(&binding.name).map_err(invalid)?; - if binding.version != 1 - || binding.owner_id != record.owner_id + if !matches!( + (binding.version, binding.lifetime.as_ref()), + (1, None) | (2, Some(_)) + ) || binding.owner_id != record.owner_id || binding.worktree_id != record.worktree_id { return Err(invalid("Snapshot checkout owner changed")); } + if let Some(owner) = &binding.lifetime { + owner.validate_fields()?; + if owner.activity_id != record.owner_id { + return Err(invalid("Snapshot checkout lifetime owner changed")); + } + } + let prefix = format!("preparing_{}_{}_", binding.owner_id.len(), binding.owner_id); + if binding.staging_name.as_ref().is_some_and(|stage| { + !stage.starts_with(&prefix) + || stage.len() != prefix.len() + 26 + || !stage + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_-".contains(&byte)) + }) || binding.version == 2 && binding.staging_name.is_none() + { + return Err(invalid("Invalid snapshot checkout construction binding")); + } regular(file)?; if identity(file)? != binding.file_identity { return Err(invalid("Snapshot checkout descriptor changed")); @@ -216,8 +284,8 @@ impl Pins { if digest(&bytes(¤t.creation)?) != binding.creation_digest { return Err(invalid("Snapshot checkout creation contents changed")); } - let observed_file = open_file(¤t.directory, &format!("{}.lock", binding.owner_id))?; - let raw = contents(file, 4096)?; + let observed_file = open_file(¤t.directory, name)?; + let raw = contents(file, super::record_limit(record))?; let observed: UseRecord = serde_json::from_slice(&raw).map_err(invalid)?; let exact: serde_json::Value = serde_json::from_slice(&raw).map_err(invalid)?; if identity(&observed_file)? != binding.file_identity diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/publication.rs b/crates/cyber-core/src/worktrees/activity/snapshot/publication.rs new file mode 100644 index 000000000..9ec18c3e3 --- /dev/null +++ b/crates/cyber-core/src/worktrees/activity/snapshot/publication.rs @@ -0,0 +1,325 @@ +//! Complete locked records precede stable publication; incomplete stages remain evidence. +use super::super::{CheckoutLease, LockedRecord, Managed, UseRecord, invalid}; +use super::{CheckoutBinding, CheckoutOwner, Pins}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions}; +use std::{fs::File, io}; + +pub(in crate::worktrees::activity) fn publish( + managed: &Managed, + activity: &str, + lifetime: Option, +) -> io::Result { + publish_before(managed, activity, lifetime, |_| Ok(())) +} +fn publish_before( + managed: &Managed, + activity: &str, + lifetime: Option, + before_publish: impl FnOnce(&CheckoutBinding) -> io::Result<()>, +) -> io::Result { + super::super::safe_id(activity, "op")?; + if let Some(owner) = &lifetime { + owner.validate(managed)?; + if owner.activity_id != activity { + return Err(invalid("Foreign snapshot checkout activity")); + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "Lifetime-bound checkout publication requires native directory durability", + )); + } + super::super::use_directory(managed, true)?; + let pins = Pins::open(&managed.common_dir, &managed.id, &managed.name)?; + let directory = &pins.directory; + let stable = format!("{activity}.lock"); + let prefix = format!("preparing_{}_{activity}_", activity.len()); + for (index, entry) in directory.entries()?.enumerate() { + if index >= 4095 { + return Err(invalid( + "Checkout activity inventory exceeds limit; acknowledged cleanup is required", + )); + } + if entry?.file_name().to_string_lossy().starts_with(&prefix) { + return Err(invalid( + "Snapshot checkout construction evidence is retained; recovery is required", + )); + } + } + match directory.symlink_metadata(&stable) { + Ok(_) => { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "Snapshot checkout claim already exists", + )); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let stage = format!("{prefix}{}", ulid::Ulid::new()); + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .create_new(true) + .follow(FollowSymlinks::No); + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = directory.open_with(&stage, &options)?.into_std(); + if !super::super::try_lock(&file)? { + return Err(invalid("Snapshot checkout construction lock is busy")); + } + let mut file = LockedRecord(file); + let mut binding = pins.capture(managed, activity, &file)?; + binding.version = if lifetime.is_some() { 2 } else { 1 }; + binding.lifetime = lifetime; + binding.staging_name = Some(stage.clone()); + let record = UseRecord { + worktree_id: managed.id.clone(), + owner_id: activity.into(), + settled: false, + snapshot_binding: Some(binding.clone()), + }; + super::super::write_record(&mut file, &record)?; + sync_file(&file)?; + sync_directory(directory)?; + pins.verify_at(&file, &record, &stage)?; + before_publish(&binding)?; + pins.verify_at(&file, &record, &stage)?; + publish_name(directory, &stage, &stable)?; + sync_directory(directory)?; + let lease = CheckoutLease { + file, + record, + snapshot_pins: Some(pins), + }; + lease.verify_snapshot()?; + Ok(lease) +} + +fn sync_file(file: &File) -> io::Result<()> { + file.sync_all()?; + #[cfg(target_os = "macos")] + rustix::fs::fcntl_fullfsync(file)?; + Ok(()) +} +fn sync_directory(directory: &Dir) -> io::Result<()> { + #[cfg(unix)] + { + let file = rustix::fs::openat( + directory, + ".", + rustix::fs::OFlags::RDONLY + | rustix::fs::OFlags::DIRECTORY + | rustix::fs::OFlags::CLOEXEC + | rustix::fs::OFlags::NOFOLLOW, + rustix::fs::Mode::empty(), + )?; + File::from(file).sync_all()?; + } + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} +fn publish_name(directory: &Dir, stage: &str, stable: &str) -> io::Result<()> { + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + rustix::fs::renameat_with( + directory, + stage, + directory, + stable, + rustix::fs::RenameFlags::NOREPLACE, + )?; + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + // No replacement. An interrupted transition retains both names and must + // fail alias verification; this is not lifetime-bound durable admission. + directory.hard_link(stage, directory, stable)?; + directory.remove_file(stage)?; + } + Ok(()) +} + +#[cfg(all(test, any(target_os = "macos", target_os = "linux")))] +mod tests { + use super::*; + use std::io::Read; + fn fixture(root: &std::path::Path) -> Managed { + let common = root.join("common"); + let path = root.join("work"); + std::fs::create_dir_all(common.join("cyber-worktrees")).unwrap(); + std::fs::create_dir_all(&path).unwrap(); + let managed = Managed { + id: "wt_fixture".into(), + name: "fixture".into(), + path, + branch: "cyber/fixture".into(), + base: "main".into(), + common_dir: common, + ready: true, + included: vec![], + }; + std::fs::write( + managed.common_dir.join("cyber-worktrees/fixture.json"), + serde_json::to_vec(&managed).unwrap(), + ) + .unwrap(); + managed + } + fn owner(managed: &Managed) -> CheckoutOwner { + CheckoutOwner { + activity_id: "op_fixture".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: managed.path.clone(), + lifetime_digest: "a".repeat(64), + } + } + fn directory(managed: &Managed) -> std::path::PathBuf { + managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id) + } + #[test] + fn complete_durable_stage_precedes_stable_name_and_a_collision_never_replaces_evidence() { + let temp = tempfile::tempdir().unwrap(); + let managed = fixture(&temp.path().canonicalize().unwrap()); + let owner = owner(&managed); + let result = publish_before( + &managed, + &owner.activity_id, + Some(owner.clone()), + |binding| { + assert!(!directory(&managed).join("op_fixture.lock").exists()); + let bytes = + std::fs::read(directory(&managed).join(binding.staging_name.as_ref().unwrap())) + .unwrap(); + let record: UseRecord = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(record.snapshot_binding.as_ref(), Some(binding)); + assert_eq!(binding.lifetime.as_ref(), Some(&owner)); + std::fs::write( + directory(&managed).join("op_fixture.lock"), + b"foreign owner bytes", + ) + .unwrap(); + Ok(()) + }, + ); + assert!(result.is_err()); + assert_eq!( + std::fs::read(directory(&managed).join("op_fixture.lock")).unwrap(), + b"foreign owner bytes" + ); + assert_eq!(std::fs::read_dir(directory(&managed)).unwrap().count(), 2); + } + #[test] + fn namespace_replacement_before_publication_retains_the_original_stage_without_installing_a_claim() + { + let temp = tempfile::tempdir().unwrap(); + let managed = fixture(&temp.path().canonicalize().unwrap()); + let owner = owner(&managed); + let retained = directory(&managed).with_extension("retained"); + let result = publish_before( + &managed, + &owner.activity_id, + Some(owner.clone()), + |binding| { + let path = directory(&managed); + let stage = binding.staging_name.as_ref().unwrap(); + let bytes = std::fs::read(path.join(stage)).unwrap(); + std::fs::rename(&path, &retained).unwrap(); + std::fs::create_dir(&path).unwrap(); + std::fs::write(path.join(stage), &bytes).unwrap(); + Ok(()) + }, + ); + assert!(result.is_err()); + assert!(!directory(&managed).join("op_fixture.lock").exists()); + assert!(!retained.join("op_fixture.lock").exists()); + let original = std::fs::read_dir(&retained) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let record: UseRecord = serde_json::from_slice(&std::fs::read(original).unwrap()).unwrap(); + assert_eq!(record.snapshot_binding.unwrap().lifetime, Some(owner)); + } + + #[test] + #[ignore = "subprocess fixture invoked by the publication kill test"] + fn snapshot_publication_child() { + let root = + std::path::PathBuf::from(std::env::var_os("CYBER_CHECKOUT_PUBLICATION_ROOT").unwrap()); + let managed = fixture(&root); + let owner = owner(&managed); + let _ = publish_before( + &managed, + &owner.activity_id, + Some(owner.clone()), + |binding| { + std::fs::write( + root.join("ready.json"), + serde_json::to_vec(binding).unwrap(), + ) + .unwrap(); + std::io::stdin().read_exact(&mut [0]).unwrap(); + Ok(()) + }, + ); + } + #[test] + fn killed_construction_keeps_locked_complete_stage_and_never_creates_or_replays_stable_claim() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "worktrees::activity::snapshot::publication::tests::snapshot_publication_child", + "--ignored", + ]) + .env("CYBER_CHECKOUT_PUBLICATION_ROOT", &root) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let binding: CheckoutBinding = loop { + if let Ok(bytes) = std::fs::read(root.join("ready.json")) + && let Ok(binding) = serde_json::from_slice(&bytes) + { + break binding; + } + assert!(child.try_wait().unwrap().is_none()); + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(2)); + }; + let managed: Managed = serde_json::from_slice( + &std::fs::read(root.join("common/cyber-worktrees/fixture.json")).unwrap(), + ) + .unwrap(); + let stage = directory(&managed).join(binding.staging_name.as_ref().unwrap()); + let file = File::open(&stage).unwrap(); + assert!(matches!( + file.try_lock(), + Err(std::fs::TryLockError::WouldBlock) + )); + child.kill().unwrap(); + child.wait().unwrap(); + assert!(file.try_lock().is_ok()); + file.unlock().unwrap(); + let retained = std::fs::read(&stage).unwrap(); + assert!(!directory(&managed).join("op_fixture.lock").exists()); + assert!(publish(&managed, "op_fixture", binding.lifetime.clone()).is_err()); + assert_eq!(std::fs::read(stage).unwrap(), retained); + let fresh = publish(&managed, "op_fresh", None).unwrap(); + fresh.verify_snapshot().unwrap(); + } +} diff --git a/crates/cyber-core/tests/worktree_repository/activity.rs b/crates/cyber-core/tests/worktree_repository/activity.rs index a7df2d275..5b6424e55 100644 --- a/crates/cyber-core/tests/worktree_repository/activity.rs +++ b/crates/cyber-core/tests/worktree_repository/activity.rs @@ -1,6 +1,58 @@ use super::*; use cyber_core::worktrees::CheckoutActivity; +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[test] +fn lifetime_bound_snapshot_claim_publishes_complete_context_without_reusing_existing_owners() { + use cyber_core::worktrees::CheckoutOwner; + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("bound-snapshot").unwrap(), + &Settings::default(), + ) + .unwrap(); + let owner = CheckoutOwner { + activity_id: "op_bound".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: managed.path.clone(), + lifetime_digest: "a".repeat(64), + }; + let lease = + block_on(repository.claim_snapshot_bound(&fixture.execution, &managed, &owner)).unwrap(); + lease.verify_snapshot().unwrap(); + assert_eq!(lease.snapshot_binding().unwrap().version, 2); + assert_eq!( + lease.snapshot_binding().unwrap().lifetime.as_ref(), + Some(&owner) + ); + assert!(lease.snapshot_binding().unwrap().staging_name.is_some()); + let directory = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), 1); + let bytes = std::fs::read(directory.join("op_bound.lock")).unwrap(); + assert!( + block_on(repository.claim_snapshot_bound(&fixture.execution, &managed, &owner)).is_err() + ); + let proof = lease.settle_retained().unwrap(); + proof.verify_snapshot().unwrap(); + drop(proof); + assert!( + block_on(repository.claim_snapshot_bound(&fixture.execution, &managed, &owner)).is_err() + ); + let record: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + record["snapshot_binding"]["lifetime"], + serde_json::to_value(owner).unwrap() + ); + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)).unwrap(); +} + #[cfg(unix)] #[test] fn snapshot_native_checkout_proof_refuses_aliases_and_an_identical_replacement_record() { diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 8e9846a33..c0537c798 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -15,6 +15,9 @@ use super::model::{CallState, RetrySafety, SessionInfo}; /// An owned Location admission. Disposal without settlement must retain recovery evidence. pub trait LocationGuard: Send { fn settle(self: Box) -> Result<(), String>; + fn snapshot_bindings(&self) -> Result, String> { + Err("Snapshot checkout binding inspection is unsupported".into()) + } /// Acknowledge native settlement and retain ownership through a consuming commit. fn settle_retained(self: Box) -> Result, String> { @@ -30,6 +33,9 @@ pub trait LocationGuard: Send { /// launch processes or perform source effects; disposal may run in the writer. pub trait SnapshotLocationGuard: Send { fn verify(&self) -> Result<(), String>; + fn bindings(&self) -> Result, String> { + Ok(vec![]) + } } struct UnmanagedSnapshotLocation { _owner: Box, @@ -46,6 +52,15 @@ pub struct LocationLease { } impl LocationLease { + pub fn snapshot_bindings(&self) -> Result, String> { + if self.worktree_id.is_none() { + return Ok(vec![]); + } + self.guard + .as_ref() + .ok_or("Missing managed checkout guard")? + .snapshot_bindings() + } pub fn guarded(worktree_id: Option, guard: Box) -> Self { Self { worktree_id, @@ -899,6 +914,22 @@ pub trait ToolHost: Send + Sync { } } + fn claim_snapshot_location_bound<'a>( + &'a self, + info: &'a SessionInfo, + activity: &'a str, + lifetime: Option<&'a super::SnapshotLifetimeBinding>, + cancel: CancellationToken, + ) -> BoxFuture<'a, Result> { + if lifetime.is_some() && info.worktree_id.is_some() { + Box::pin(async { + Err("Lifetime-bound snapshot checkout ownership is unsupported".into()) + }) + } else { + self.claim_snapshot_location(info, activity, cancel) + } + } + /// Prepare an existing child under its exclusive owner. Opening a client view grants no bypass. fn prepare_child_continuation<'a>( &'a self, diff --git a/crates/cyber-server/src/runtime/location.rs b/crates/cyber-server/src/runtime/location.rs index 225257fd3..d74d553f7 100644 --- a/crates/cyber-server/src/runtime/location.rs +++ b/crates/cyber-server/src/runtime/location.rs @@ -23,11 +23,15 @@ impl Inner { &self, info: &SessionInfo, activity: &str, + lifetime: Option<&super::SnapshotLifetimeBinding>, cancel: CancellationToken, ) -> Result { - let claim = self - .tools - .claim_snapshot_location(info, activity, cancel.child_token()); + let claim = self.tools.claim_snapshot_location_bound( + info, + activity, + lifetime, + cancel.child_token(), + ); self.await_location_claim(claim, cancel).await } diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs index 9605f1b62..beaf04bda 100644 --- a/crates/cyber-server/src/runtime/snapshot_lifetime.rs +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -25,31 +25,21 @@ pub struct SnapshotLifetimeBinding { pub source_phase: Option, } -/// The actual worker retains this lock through all resource acquisition and work. -/// It conveys no authority to settle checkout, process or binary-document owners. -pub trait SnapshotLifetime: Send { - fn binding(&self) -> &SnapshotLifetimeBinding; - /// Synchronous native metadata verification; may run inside the Store writer. - /// It must not call the runtime/Store, launch processes or perform source effects. - fn verify(&self) -> Result<(), String>; - /// Persist native admission before a source callback. A legacy lifetime may - /// never grant the new positive pre-effect cancellation proof. - fn admit_source(&self, _origin: &super::SnapshotOrigin) -> Result<(), String> { - Err("Native source admission is unavailable".into()) - } - /// Positive native source-phase proof, not merely an unlocked lifetime file. - fn verify_preparation(&self, _origin: Option<&super::SnapshotOrigin>) -> Result<(), String> { - Err("Native preparation phase proof is unavailable".into()) - } - /// Metadata publication only, after observed durable preparation cancellation. - fn acknowledge_preparation( - &mut self, - receipt: &super::SnapshotPreparationReceipt, - ) -> Result<(), String> { - if receipt.origin.is_some() { - return Err("Preparation journal acknowledgement is unavailable".into()); +impl SnapshotLifetimeBinding { + pub fn checkout_owner(&self) -> Result { + use sha2::{Digest, Sha256}; + if self.version != 2 || self.source_phase.is_none() { + return Err("Legacy lifetime has no checkout binding authority".into()); } - Ok(()) + let bytes = serde_json::to_vec(&("snapshot-checkout-lifetime-v1", self)) + .map_err(|error| error.to_string())?; + Ok(cyber_core::worktrees::CheckoutOwner { + activity_id: self.owner.activity_id.clone(), + session_id: self.owner.session_id.clone(), + session_created_id: self.owner.session_created_id.clone(), + directory: self.owner.directory.clone(), + lifetime_digest: format!("{:x}", Sha256::digest(bytes)), + }) } } @@ -114,3 +104,59 @@ impl super::Runtime { Ok(lifetime) } } + +/// The actual worker retains this lock through all resource acquisition and work. +/// It conveys no authority to settle checkout, process or binary-document owners. +pub trait SnapshotLifetime: Send { + fn binding(&self) -> &SnapshotLifetimeBinding; + /// Synchronous native metadata verification; may run inside the Store writer. + /// It must not call the runtime/Store, launch processes or perform source effects. + fn verify(&self) -> Result<(), String>; + /// Persist native admission before a source callback. A legacy lifetime may + /// never grant the new positive pre-effect cancellation proof. + fn admit_source(&self, _origin: &super::SnapshotOrigin) -> Result<(), String> { + Err("Native source admission is unavailable".into()) + } + /// Positive native source-phase proof, not merely an unlocked lifetime file. + fn verify_preparation(&self, _origin: Option<&super::SnapshotOrigin>) -> Result<(), String> { + Err("Native preparation phase proof is unavailable".into()) + } + /// Metadata publication only, after observed durable preparation cancellation. + fn acknowledge_preparation( + &mut self, + receipt: &super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + if receipt.origin.is_some() { + return Err("Preparation journal acknowledgement is unavailable".into()); + } + Ok(()) + } +} + +#[cfg(test)] +mod checkout_encoding_tests { + use super::*; + #[test] + fn checkout_digest_encoding_has_a_stable_versioned_known_answer() { + let binding = SnapshotLifetimeBinding { + version: 2, + owner: SnapshotLifetimeRequest { + activity_id: "op_fixture".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: "/work".into(), + worktree_id: None, + }, + root_identity: "root".into(), + namespace_identity: "namespace".into(), + directory_identity: "directory".into(), + file_identity: "file".into(), + source_phase: Some("phase_fixture".into()), + }; + // Pins the framed v1 encoding, rather than recomputing it with this helper. + assert_eq!( + binding.checkout_owner().unwrap().lifetime_digest, + "66239118d7327f0739d4e468da0b7d4c547e0a752cfd9680e6e5f7f2469af802" + ); + } +} diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 242f43909..4e777444b 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -246,6 +246,47 @@ impl SnapshotWork<'_> { } impl SnapshotWorker { + fn checkout_proof( + scope: &activity::Scope, + lease: super::LocationLease, + worktree: Option<&str>, + ) -> Result, RuntimeError> { + let bindings = lease.snapshot_bindings().map_err(RuntimeError::Conflict)?; + let expected = scope + .snapshot_lifetime() + .map(super::SnapshotLifetimeBinding::checkout_owner) + .transpose() + .map_err(RuntimeError::Conflict)?; + if bindings.len() > 4096 + || worktree.is_some_and(|id| !bindings.iter().any(|binding| binding.worktree_id == id)) + { + return Err(RuntimeError::Conflict( + "Snapshot checkout resource proof is incomplete or exceeds limits".into(), + )); + } + let mut seen = std::collections::HashSet::new(); + for binding in &bindings { + if binding.version != if expected.is_some() { 2 } else { 1 } + || binding.owner_id != scope.id() + || binding.lifetime != expected + || !seen.insert((&binding.common_dir, &binding.worktree_id)) + { + return Err(RuntimeError::Conflict( + "Snapshot checkout proof has foreign lifetime or ownership".into(), + )); + } + } + let proof = lease + .settle_snapshot_retained() + .map_err(RuntimeError::Invalid)?; + proof.verify().map_err(RuntimeError::Conflict)?; + if proof.bindings().map_err(RuntimeError::Conflict)? != bindings { + return Err(RuntimeError::Conflict( + "Snapshot checkout binding changed during retained settlement".into(), + )); + } + Ok(proof) + } fn location(&self) -> Result<&dyn super::SnapshotLocationGuard, String> { self._location .as_deref() @@ -405,7 +446,12 @@ impl Runtime { )?; let lease = self .inner - .claim_snapshot_location(&state.info, scope.id(), scope.control.stop.clone()) + .claim_snapshot_location( + &state.info, + scope.id(), + scope.snapshot_lifetime(), + scope.control.stop.clone(), + ) .await?; let directory = std::fs::canonicalize(&state.info.directory) .map_err(|error| RuntimeError::Invalid(error.to_string()))?; @@ -426,6 +472,8 @@ impl Runtime { "Recovery native Location/creation binding changed".into(), )); } + let location = + SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; Ok(SnapshotWorker { scope, admission, @@ -434,11 +482,7 @@ impl Runtime { directory, created_id, worktree_id: state.info.worktree_id.clone(), - _location: Some( - lease - .settle_snapshot_retained() - .map_err(RuntimeError::Invalid)?, - ), + _location: Some(location), _lifecycle: lifecycle, _lifetime: lifetime, }) @@ -469,7 +513,12 @@ impl Runtime { .await?; let lease = self .inner - .claim_snapshot_location(&state.info, scope.id(), scope.control.stop.clone()) + .claim_snapshot_location( + &state.info, + scope.id(), + scope.snapshot_lifetime(), + scope.control.stop.clone(), + ) .await?; if lease.worktree_id != state.info.worktree_id { return Err(RuntimeError::Conflict( @@ -488,17 +537,15 @@ impl Runtime { .id .clone(); let worktree_id = state.info.worktree_id.clone(); + let location = + SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; Ok(SnapshotWorker { scope, runtime: self.clone(), directory, created_id, worktree_id, - _location: Some( - lease - .settle_snapshot_retained() - .map_err(RuntimeError::Invalid)?, - ), + _location: Some(location), _lifecycle: lifecycle, _lifetime: lifetime, admission, diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 52b28b2f5..5403a824d 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -190,6 +190,118 @@ async fn default_host_refuses_managed_snapshot_claim_without_explicit_support() } struct Acknowledgements(Arc); + +struct BindingLease { + bindings: Vec, + settlements: Arc, +} +impl super::super::LocationGuard for BindingLease { + fn settle(self: Box) -> Result<(), String> { + self.settlements.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + fn snapshot_bindings(&self) -> Result, String> { + Ok(self.bindings.clone()) + } + fn settle_snapshot_retained( + self: Box, + ) -> Result, String> { + self.settlements.fetch_add(1, Ordering::SeqCst); + Ok(self) + } +} +impl super::super::SnapshotLocationGuard for BindingLease { + fn verify(&self) -> Result<(), String> { + Ok(()) + } + fn bindings(&self) -> Result, String> { + Ok(self.bindings.clone()) + } +} + +#[tokio::test] +async fn checkout_lifetime_context_is_checked_before_any_native_settlement() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + }), + ); + let owner = worker(&runtime, &session).await; + let lifetime = owner.scope.snapshot_lifetime().unwrap(); + let context = lifetime.checkout_owner().unwrap(); + let mut changed = lifetime.clone(); + changed.file_identity.push_str("_replacement"); + assert_ne!( + changed.checkout_owner().unwrap().lifetime_digest, + context.lifetime_digest + ); + changed.version = 1; + assert!(changed.checkout_owner().is_err()); + let binding = cyber_core::worktrees::CheckoutBinding { + version: 2, + owner_id: owner.scope.id().into(), + worktree_id: "wt_fixture".into(), + name: "fixture".into(), + common_dir: temp.path().to_path_buf(), + common_identity: "fixture".into(), + root_identity: "fixture".into(), + directory_identity: "fixture".into(), + file_identity: "fixture".into(), + creation_directory_identity: "fixture".into(), + creation_file_identity: "fixture".into(), + creation_digest: "a".repeat(64), + lifetime: Some(context), + staging_name: Some("preparing_fixture".into()), + }; + for case in [ + "valid", + "activity", + "creation", + "digest", + "missing", + "version", + "duplicate", + "empty", + ] { + let mut bindings = vec![binding.clone()]; + match case { + "activity" => bindings[0].owner_id = "op_foreign".into(), + "creation" => { + bindings[0].lifetime.as_mut().unwrap().session_created_id = "evt_foreign".into() + } + "digest" => bindings[0].lifetime.as_mut().unwrap().lifetime_digest = "b".repeat(64), + "missing" => bindings[0].lifetime = None, + "version" => bindings[0].version = 3, + "duplicate" => bindings.push(binding.clone()), + "empty" => bindings.clear(), + _ => {} + } + let settlements = Arc::new(AtomicUsize::new(0)); + let lease = super::super::LocationLease::managed( + "wt_fixture".into(), + Box::new(BindingLease { + bindings, + settlements: settlements.clone(), + }), + ); + let result = SnapshotWorker::checkout_proof(&owner.scope, lease, Some("wt_fixture")); + assert_eq!(result.is_ok(), case == "valid", "{case}"); + assert_eq!( + settlements.load(Ordering::SeqCst), + usize::from(case == "valid"), + "{case}" + ); + } + drop(owner); + runtime.shutdown().await; + base.shutdown().await; +} struct CheckoutProof { calls: Arc, fail_at: Arc, diff --git a/crates/cyber-store/migrations/20261106030000_snapshot_checkout_lifetime.sql b/crates/cyber-store/migrations/20261106030000_snapshot_checkout_lifetime.sql new file mode 100644 index 000000000..dec6aa071 --- /dev/null +++ b/crates/cyber-store/migrations/20261106030000_snapshot_checkout_lifetime.sql @@ -0,0 +1,3 @@ +-- Lifetime-bound checkout publication and pre-settlement context verification +-- require a writer that understands the complete native ownership contract. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index feaae4ae9..bb1564057 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -185,6 +185,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261106020000_snapshot_checkout_proof", sql: include_str!("../migrations/20261106020000_snapshot_checkout_proof.sql"), }, + Migration { + id: "20261106030000_snapshot_checkout_lifetime", + sql: include_str!("../migrations/20261106030000_snapshot_checkout_lifetime.sql"), + }, ]; pub(crate) enum Outcome { @@ -398,10 +402,49 @@ mod tests { let Outcome::TooNew(unknown) = migrate(&mut db, previous).unwrap() else { panic!("previous writers bypassed checkout proof admission"); }; - assert_eq!(unknown, ["20261106020000_snapshot_checkout_proof"]); + assert_eq!( + unknown, + MIGRATIONS[at..] + .iter() + .map(|migration| migration.id) + .collect::>() + ); assert_eq!(std::fs::read(claim).unwrap(), evidence); } + #[test] + fn checkout_lifetime_catalog_rejects_activity_only_writers_and_preserves_evidence() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("lifetime.db"); + let at = MIGRATIONS + .iter() + .position(|migration| migration.id == "20261106030000_snapshot_checkout_lifetime") + .unwrap(); + let mut db = Connection::open(&path).unwrap(); + assert!(matches!( + migrate(&mut db, &MIGRATIONS[..at]).unwrap(), + Outcome::Current + )); + let evidence = r#"{"session_id":"ses_retained","snapshot_worker":true,"status":"unknown","snapshot_lifetime":{"version":2}}"#; + db.execute("INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('evt_context','op_retained',0,'native.activity.changed.1',?1,1)", [evidence]).unwrap(); + assert!(matches!( + migrate(&mut db, MIGRATIONS).unwrap(), + Outcome::Current + )); + drop(db); + let mut db = Connection::open(path).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { + panic!("activity-only writer accepted lifetime-bound checkout storage"); + }; + assert_eq!(unknown, ["20261106030000_snapshot_checkout_lifetime"]); + let retained: String = db + .query_row("SELECT data FROM event WHERE id='evt_context'", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(retained, evidence); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index 4b65949d7..c32c7fad2 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1362,6 +1362,16 @@ impl ToolHost for BuiltinHost { Box::pin(self.claim_snapshot_worktree_location(info, activity, cancel)) } + fn claim_snapshot_location_bound<'a>( + &'a self, + info: &'a cyber_server::runtime::SessionInfo, + activity: &'a str, + lifetime: Option<&'a cyber_server::runtime::SnapshotLifetimeBinding>, + cancel: CancellationToken, + ) -> BoxFuture<'a, Result> { + Box::pin(self.claim_snapshot_worktree_location_bound(info, activity, lifetime, cancel)) + } + fn prepare_child_continuation<'a>( &'a self, parent: &'a cyber_server::runtime::SessionInfo, diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index e2b029ebd..04359b2a1 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -36,6 +36,7 @@ use crate::tools::process::Process; enum CheckoutWorker<'a> { Binary(&'a str), Snapshot(&'a str), + SnapshotBound(&'a cyber_core::worktrees::CheckoutOwner), } /// Inputs for creating a new Session in a managed worktree. @@ -247,6 +248,41 @@ impl BuiltinHost { .await } + pub(crate) async fn claim_snapshot_worktree_location_bound( + &self, + info: &SessionInfo, + activity: &str, + lifetime: Option<&cyber_server::runtime::SnapshotLifetimeBinding>, + cancel: CancellationToken, + ) -> Result { + let owner = lifetime + .map(|lifetime| { + if lifetime.owner.activity_id != activity + || lifetime.owner.session_id != info.id + || lifetime.owner.worktree_id != info.worktree_id + || std::fs::canonicalize(&info.directory).map_err(|error| error.to_string())? + != lifetime.owner.directory + { + return Err("Foreign snapshot checkout lifetime".into()); + } + lifetime.checkout_owner() + }) + .transpose()?; + let inv = location_invocation(info); + let worker = owner + .as_ref() + .map(CheckoutWorker::SnapshotBound) + .unwrap_or(CheckoutWorker::Snapshot(activity)); + self.claim_location( + &inv, + info.worktree_id.as_deref(), + false, + cancel, + Some(worker), + ) + .await + } + pub(crate) async fn claim_hook_test_location( &self, event: &cyber_core::hooks::HookEvent, @@ -342,6 +378,11 @@ impl BuiltinHost { .claim_snapshot(&execution, &managed, activity) .await } + Some(CheckoutWorker::SnapshotBound(owner)) => { + repository + .claim_snapshot_bound(&execution, &managed, owner) + .await + } None => { repository .claim(&execution, &managed, &inv.session_id) @@ -1204,6 +1245,9 @@ struct ActiveLocation { activity: crate::lsp::Activity, } impl cyber_server::runtime::LocationGuard for ActiveLocation { + fn snapshot_bindings(&self) -> Result, String> { + self.lease.snapshot_bindings() + } fn settle(self: Box) -> Result<(), String> { self.lease.settle() } @@ -1231,6 +1275,9 @@ impl cyber_server::runtime::SnapshotLocationGuard for ActiveSnapshotLocation { fn verify(&self) -> Result<(), String> { self.proof.verify() } + fn bindings(&self) -> Result, String> { + self.proof.bindings() + } } struct SnapshotWorktreeProof(Vec); @@ -1241,9 +1288,33 @@ impl cyber_server::runtime::SnapshotLocationGuard for SnapshotWorktreeProof { } Ok(()) } + fn bindings(&self) -> Result, String> { + self.0 + .iter() + .map(|lease| { + lease.verify_snapshot().map_err(|error| error.to_string())?; + lease + .snapshot_binding() + .cloned() + .map_err(|error| error.to_string()) + }) + .collect() + } } impl cyber_server::runtime::LocationGuard for WorktreeLease { + fn snapshot_bindings(&self) -> Result, String> { + self.0 + .iter() + .map(|lease| { + lease.verify_snapshot().map_err(|error| error.to_string())?; + lease + .snapshot_binding() + .cloned() + .map_err(|error| error.to_string()) + }) + .collect() + } fn settle(self: Box) -> Result<(), String> { for lease in self.0 { lease.settle().map_err(|error| error.to_string())?; diff --git a/crates/cyber-tools/tests/worktree_setup.rs b/crates/cyber-tools/tests/worktree_setup.rs index e2f0c77ef..07d7ac43b 100644 --- a/crates/cyber-tools/tests/worktree_setup.rs +++ b/crates/cyber-tools/tests/worktree_setup.rs @@ -1912,6 +1912,90 @@ async fn process_launch_failure_remains_uncertain_and_cannot_authorize_retry() { assert!(!managed.path.join("target.txt").exists()); } +#[tokio::test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn lifetime_bound_checkout_host_preserves_context_and_refuses_foreign_requests_before_creation() + { + use cyber_server::runtime::{ + CreateSession, NoSnapshots, SnapshotLifetimeBinding, SnapshotLifetimeRequest, ToolHost, + }; + let (fixture, _, managed) = owned().await; + let flow = support::flow::Flow::with(fixture, vec![], false, Arc::new(NoSnapshots)); + let info = flow + .runtime + .create_session(CreateSession { + directory: managed.path.display().to_string(), + model: "test/main".into(), + mode: Some("bypass".into()), + ..Default::default() + }) + .await + .unwrap(); + let created = flow.f.store.read_events(&info.id, -1, 1).unwrap().events[0] + .id + .clone(); + // Adapter fixture: the runtime's separate lifetime lock proof is tested in + // runtime/native suites. This test exercises actual managed checkout creation. + let lifetime = SnapshotLifetimeBinding { + version: 2, + owner: SnapshotLifetimeRequest { + activity_id: "op_bound".into(), + session_id: info.id.clone(), + session_created_id: created, + directory: managed.path.clone(), + worktree_id: Some(managed.id.clone()), + }, + root_identity: "fixture-root".into(), + namespace_identity: "fixture-namespace".into(), + directory_identity: "fixture-directory".into(), + file_identity: "fixture-file".into(), + source_phase: Some("phase_fixture".into()), + }; + let directory = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + let before = std::fs::read_dir(&directory).unwrap().count(); + assert!( + flow.f + .host + .claim_snapshot_location_bound( + &info, + "op_foreign", + Some(&lifetime), + CancellationToken::new() + ) + .await + .is_err() + ); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), before); + let lease = flow + .f + .host + .claim_snapshot_location_bound(&info, "op_bound", Some(&lifetime), CancellationToken::new()) + .await + .unwrap(); + let bindings = lease.snapshot_bindings().unwrap(); + assert_eq!(bindings.len(), 1); + assert_eq!( + bindings[0].lifetime, + Some(lifetime.checkout_owner().unwrap()) + ); + let proof = lease.settle_snapshot_retained().unwrap(); + proof.verify().unwrap(); + assert_eq!(proof.bindings().unwrap(), bindings); + drop(proof); + let raw: serde_json::Value = + serde_json::from_slice(&std::fs::read(directory.join("op_bound.lock")).unwrap()).unwrap(); + assert_eq!( + raw["snapshot_binding"], + serde_json::to_value(&bindings[0]).unwrap() + ); + assert_eq!(raw["settled"], true); + assert_eq!(std::fs::read_dir(directory).unwrap().count(), before + 1); + flow.runtime.shutdown().await; +} + #[tokio::test] async fn managed_git_legacy_rewind_preserves_an_independent_native_checkout_proof() { use cyber_server::runtime::{CreateSession, RevertTarget, ToolHost}; diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 98cd87412..c98a68c33 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -171,3 +171,11 @@ The actual worker and consuming Store writer verify and retain that proof; alias replacement and changed records refuse. Binding publication before interrupted acquisition and recovery-owned native acknowledgement remain required. Git rewind still follows its legacy path; compatibility tests do not establish owned Git recovery. + + +The local initialized-publication continuation adds staged no-replace checkout +claims and exact lifetime context. Runtime verifies complete managed bindings before +native settlement; prior writer catalogs refuse mutation. Construction kills retain +evidence and refuse replay. Unknown-claim acknowledgement and reclamation remain +incomplete, and Windows lifetime-bound durability is not yet admitted. This does not +accept the complete recovery feature or either issue. diff --git a/docs/rewind.md b/docs/rewind.md index 05fb73df0..ea02ae9b7 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -303,3 +303,12 @@ and worktree creation metadata before source changes and consuming commits. Alia or replaced claim files, changed creation contents and replaced activity directories refuse. These checks preserve evidence; they do not grant permission to clear an unknown checkout claim after interrupted acquisition. + + +New Linux/macOS lifetime-bound checkout claims publish their complete native record +before becoming visible under the activity name. A pre-publication crash retains a +construction record and prevents reuse of that activity. Runtime validates Session +creation, directory, lifetime digest and the whole returned resource set before +settling native ownership. Construction evidence is retained for the pending safe +recovery/cleanup path; it cannot be removed to force a retry. Windows does not yet +admit the lifetime-bound durability path. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index fe86ca477..52105b751 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1312,3 +1312,80 @@ and no journal acknowledgement. Checked owner refusals return `Conflict` while schema, corruption and storage failures retain their distinct error paths. The final snapshot-worker suite (34 tests) and server Clippy passed after this change. The checkout-proof migration has a separate passing Store all-target Clippy check. + + +### Initialized lifetime-bound checkout publication (2026-11-06, local) + +Stable checkout activity names no longer expose an initial generic record followed +by a later binding rewrite. Construction stages and flushes the complete native +record while retaining its actual file lock, directory pins and creation descriptor, +then publishes without replacement and synchronizes the directory. Linux/macOS use +native no-replace rename. A pre-publication cut leaves a private complete stage; +that activity cannot be reused while its construction evidence remains. Prefixes +include activity length to prevent overlapping IDs from colliding. Admission reserves +room for the new record within a 4096-entry inventory; legacy/unbound records remain +bounded to 4 KiB, and lifetime-bound records to 16 KiB. No construction cleanup is +enabled here. + +Version-2 native claims include the exact activity, Session creation, canonical +source directory and versioned digest of the complete immutable lifetime binding. +The v1 digest framing has a fixed known-answer regression. Native constructors +validate Location/context before publication. Runtime inspects the entire returned +resource set before native settlement, rejecting foreign activity/creation/digest, +missing context, unknown versions, duplicate or incomplete resources. It checks the +same binding set after retaining ownership. This prevents a foreign resource from +being acknowledged merely to inspect it. Production CLI and App adapters implement +the bound port; unsupported managed hosts refuse rather than dropping the context. +The new compatibility marker rejects earlier writer catalogs without changing any +retained native/event bytes. + +Native regressions include actual process death while a complete stage is locked, +no stable publication at that cut, no replay/reuse of construction evidence, a +foreign stable-name collision, and namespace replacement immediately before +publication. The last retains the original stage and refuses installation. Review +keeps the original directory/creation pins through construction instead of reopening +them after creating a stage, and rechecks routing after the preparation boundary. +Production-host tests exercise actual managed creation, context roundtrip and refusal +of a foreign request before namespace creation. Scope tests use an instrumented port +to prove all context failures precede native settlement. These are acquired-resource +and publication checks, not completed unknown-claim acknowledgement. + +Windows retains initialized unbound legacy publication through no-replace link/unlink; +an interrupted extra link refuses verification. Lifetime-bound directory durability +is explicitly unavailable there and refuses before claim creation. This does not +claim Windows native acceptance or weaken an existing native fence. Native Windows +execution, Linux execution, complete publication/cut coverage, exact unknown-claim +reacquisition/acknowledgement, Git journals, reference-aware reclamation and full +client acceptance remain required. M2–M8 and parent tasks remain open. + + +Verification on local macOS arm64: all 410 core unit cases passed with two ignored +subprocess fixtures, and all 71 core worktree repository cases passed with its child +fixture listed ignored. After final pin/route review, all eight activity unit cases +and six selected native snapshot integration cases passed; the new process-kill child +is invoked by its parent test. All 422 server unit cases passed with one ignored +fixture, plus 22 affected integrations (4 idle cancellation, 13 Location activity, +5 rewind preview). The subsequent 40-case snapshot-focused run passed, including the +known-answer digest and pre-settlement context refusal cases. All 40 production +worktree host cases, all 61 Store cases and all 16 real CLI branch/recovery cases +passed. All-target core/server/Store/tools/App/CLI Clippy passed with warnings denied +after review fixes. Formatting, diff and generated SDK consistency checks passed; +strict pinned OpenSpec passed 117 items and lint reports 0 errors/25 warnings. +SDK execution tests, full-workspace tests, native Linux/Windows execution and live +TUI/App recovery acceptance were not run for this increment. Native sandbox +API deprecation and Darwin linker warnings were observed without attributing their +origin to main. + +The code-review skill was applied to every changed native/core, runtime/port, +Store compatibility, CLI/App adapter, test and documentation file. Review checks +complete pre-publication records, original pin retention and no-follow routing, +no-replace/collision behavior, locked-file durability, bounded inventory and encoding, +context identity before acknowledgement, stable digest framing, prior-catalog refusal, +and absence of source replay or legacy authority promotion. It adds revalidation at +the publication boundary and a namespace-replacement regression. Initial checks found +a borrow error in the new subprocess fixture and Clippy test placement/borrow style +issues; those are corrected and the final checks above passed. No critical/major +findings remain for initialized lifetime-bound publication. Full unknown-resource +reconciliation, complete ownership/reference publication across all cuts, Git recovery, +reclamation and platform/client acceptance remain open. This local continuation does +not make the complete recovery increment merge-ready or complete either issue. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 4801cb65c..050f7a4f6 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -332,3 +332,39 @@ receipts are still required. No old Session or operation claim is reclassified. The checkout-proof compatibility migration prevents earlier writer catalogs from opening this Store for mutation and bypassing these native verification steps. It performs no native evidence repair or deletion. + + +### Atomic initialized checkout publication and lifetime context + +New lifetime-bound claims carry a version-2 native binding containing the activity, +Session creation, canonical source directory and a domain-separated digest of the +complete immutable snapshot lifetime binding. Runtime admission validates every +returned managed resource against that exact context, not just an activity label. +Legacy claims remain distinct and gain no new acknowledgement authority. + +Construction prepares and flushes the complete bounded record under its actual +native lock before publishing the stable activity name without replacement. A cut +before publication retains private construction evidence; no partially initialized +stable record is a valid claim. Linux/macOS use no-replace native rename; the Windows +publication path must retain an interrupted link/unlink transition as evidence and +never treat a remaining alias as a completed acquisition. Proof still verifies +creation/namespace identities and exact record contents after publication. This +step does not enable interrupted-claim acknowledgement or reclaim construction +evidence; those require consuming recovery receipts and complete reference/lease +checks in the following implementation. + + +Publication inventories are bounded to 4096 entries, including the new stable claim; +exhaustion retains all evidence and requires acknowledged cleanup. Bound native +records are limited to 16 KiB; unbound/generic records retain their 4 KiB limit. +Construction prefixes include the activity length to avoid prefix collisions between +valid activity IDs. A retained preparation refuses reuse of that activity even when +its stable name was never published. Reconciliation/cleanup of those preparations +is not enabled by publication alone. + +Lifetime-bound durable publication currently admits Linux/macOS only. Windows keeps +legacy unbound claim compatibility through initialized no-replace link/unlink +publication, while a retained extra link refuses verification. It does not gain +lifetime-bound durability or recovery authority; that path refuses before creating +claim evidence until native directory durability has been implemented and verified. +This remains part of the explicit Windows/platform acceptance limits. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index e4e01a694..e9aa933ff 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -99,6 +99,12 @@ - **AND** settling that snapshot claim SHALL NOT acknowledge unrelated Session activity; an interrupted acquisition SHALL retain unknown evidence without a generic clearing path - **AND** a managed host without explicit snapshot ownership support SHALL refuse instead of reusing a Session claim +#### Scenario: Checkout claim publication is fully initialized +- **WHEN** a lifetime-bound snapshot worker acquires a checkout claim +- **THEN** the stable native claim SHALL be published without replacement only after its complete record is durable and binds the exact activity, Session creation, directory and immutable lifetime digest +- **AND** runtime admission SHALL refuse returned resources with foreign or missing lifetime context +- **AND** a construction crash SHALL preserve private evidence without granting an incomplete stable record or legacy claim new acknowledgement authority + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index b7ba41884..5782c9dab 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -198,3 +198,15 @@ the Store writer through commit/reply. Changed routes/records refuse without sou callbacks or acknowledgement. This does not finish immutable lifetime/resource publication or unknown-claim reacquisition across early cuts. Those and complete platform/client acceptance keep M2–M8 and parent tasks open. + + +Atomic lifetime-bound checkout publication continuation: new managed claims stage +and flush complete ownership records under their native lock before no-replace +publication. Version-2 claims carry exact Session creation/directory and immutable +lifetime digest. Runtime checks the complete returned resource set, including +version, foreign/missing context and duplicates, before native settlement. Read-only +inspection precedes any acknowledgement. Real native pre-publication kill and +collision tests retain evidence and refuse activity reuse. Unbound legacy claims +and Windows durability limits remain distinct. Unknown-claim acknowledgement, +full publication/recovery cuts, Git journals and reclamation remain incomplete; +no parent M2–M8 or 2.4/3.2–3.4 task is accepted here. From d8775262e417782723dc7b467c8dc56009cfa220 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 00:40:24 -0400 Subject: [PATCH 16/87] Persist complete checkout resources before snapshot source admission --- crates/cyber-cli/tests/branches_cli.rs | 15 ++ crates/cyber-core/src/worktrees.rs | 4 +- crates/cyber-core/src/worktrees/activity.rs | 15 +- .../src/worktrees/activity/snapshot.rs | 63 +++++ .../tests/worktree_repository/activity.rs | 75 ++++++ crates/cyber-server/src/runtime/activity.rs | 25 +- .../src/runtime/snapshot_lifetime.rs | 14 + .../cyber-server/src/runtime/snapshot_work.rs | 10 + .../src/runtime/snapshot_work_tests.rs | 66 +++++ crates/cyber-snapshot/src/copy/lifetime.rs | 239 ++++++++++++++++-- ...1107000000_snapshot_checkout_resources.sql | 3 + crates/cyber-store/src/migrate.rs | 41 ++- docs/goal-delivery.md | 8 + .../implementation.md | 74 ++++++ .../managed-recovery.md | 20 ++ .../specs/snapshots-checkpoints/spec.md | 11 + .../complete-daily-coding-surfaces/tasks.md | 12 + 17 files changed, 665 insertions(+), 30 deletions(-) create mode 100644 crates/cyber-store/migrations/20261107000000_snapshot_checkout_resources.sql diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 6f3b7af41..cfbea2900 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1395,6 +1395,21 @@ fn assert_snapshot_lifetime(namespace: &std::path::Path, activity: &Value, origi let recorded_lifetime: Value = serde_json::from_slice(&std::fs::read(lifetime_file).unwrap()).unwrap(); assert_eq!(recorded_lifetime, *lifetime); + let phase: Value = serde_json::from_slice( + &std::fs::read( + namespace + .join("workers") + .join(lifetime["source_phase"].as_str().unwrap()), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(phase["lifetime"], *lifetime); + assert_eq!(phase["admitted"], true); + assert_eq!(phase["origin"], *origin); + // These real-service fixtures use unmanaged Locations. An explicitly + // published empty set is distinct from missing constructor evidence. + assert_eq!(phase["checkout_bindings"], serde_json::json!([])); } #[test] diff --git a/crates/cyber-core/src/worktrees.rs b/crates/cyber-core/src/worktrees.rs index f85460f8a..6c6adc583 100644 --- a/crates/cyber-core/src/worktrees.rs +++ b/crates/cyber-core/src/worktrees.rs @@ -8,7 +8,9 @@ mod location; mod removal; pub(crate) mod repository; mod setup; -pub use activity::{CheckoutActivity, CheckoutBinding, CheckoutLease, CheckoutOwner}; +pub use activity::{ + CheckoutActivity, CheckoutBinding, CheckoutLease, CheckoutOwner, SnapshotClaim, +}; pub use changes::{ChangedFile, Changes}; pub use includes::IncludedFile; pub use removal::{RemovalActivity, RemovalPhase, RemovalRecord}; diff --git a/crates/cyber-core/src/worktrees/activity.rs b/crates/cyber-core/src/worktrees/activity.rs index 050ece170..9b4b960ec 100644 --- a/crates/cyber-core/src/worktrees/activity.rs +++ b/crates/cyber-core/src/worktrees/activity.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use super::{GitExecution, Managed, Name, RemovalActivity, Repository, RepositoryLock}; mod snapshot; -pub use snapshot::{CheckoutBinding, CheckoutOwner}; +pub use snapshot::{CheckoutBinding, CheckoutOwner, SnapshotClaim}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct UseRecord { @@ -138,6 +138,19 @@ impl RemovalActivity for CheckoutActivity { } impl Repository { + /// Reacquire exact existing evidence, without creating or settling a claim. + /// Caller must separately retain the matching snapshot lifetime proof. + pub fn reacquire_snapshot( + &self, + managed: &Managed, + binding: &CheckoutBinding, + owner: &CheckoutOwner, + ) -> io::Result { + if self.common_dir != managed.common_dir { + return Err(invalid("Foreign checkout repository")); + } + snapshot::reacquire(managed, binding, owner) + } /// Admit Location use under the same lock as removal and verify exact ready /// ownership before creating a persistent per-Session activity lock. pub async fn claim( diff --git a/crates/cyber-core/src/worktrees/activity/snapshot.rs b/crates/cyber-core/src/worktrees/activity/snapshot.rs index cc73dfa69..2b23129b8 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot.rs @@ -42,6 +42,69 @@ pub struct CheckoutOwner { pub directory: PathBuf, pub lifetime_digest: String, } + +/// Read-only reacquired evidence, deliberately separate from `CheckoutLease`. +/// No generic settlement, creation or source mutation capability is exposed. +pub struct SnapshotClaim { + lease: super::CheckoutLease, +} +impl SnapshotClaim { + pub fn binding(&self) -> &CheckoutBinding { + self.lease + .record + .snapshot_binding + .as_ref() + .expect("reacquired binding") + } + pub fn verify(&self) -> io::Result<()> { + self.lease.verify_snapshot() + } +} + +pub(super) fn reacquire( + managed: &Managed, + binding: &CheckoutBinding, + owner: &CheckoutOwner, +) -> io::Result { + super::safe_id(&managed.id, "wt")?; + super::super::Name::parse(&managed.name).map_err(invalid)?; + owner.validate(managed)?; + if binding.version != 2 + || binding.lifetime.as_ref() != Some(owner) + || binding.owner_id != owner.activity_id + || binding.worktree_id != managed.id + || binding.name != managed.name + || binding.common_dir != managed.common_dir + { + return Err(invalid( + "Checkout reacquisition differs from its exact lifetime or creation", + )); + } + let pins = Pins::open(&binding.common_dir, &binding.worktree_id, &binding.name)?; + pins.matches(binding)?; + let observed: Managed = serde_json::from_slice(&bytes(&pins.creation)?).map_err(invalid)?; + if observed != *managed || !observed.ready { + return Err(invalid( + "Snapshot checkout creation differs from reacquisition", + )); + } + let file = open_file(&pins.directory, &format!("{}.lock", binding.owner_id))?; + if !super::try_lock(&file)? { + return Err(invalid("Snapshot checkout worker is still active")); + } + let mut file = super::LockedRecord(file); + let record = super::read_record(&mut file)?; + if record.snapshot_binding.as_ref() != Some(binding) { + return Err(invalid("Snapshot checkout binding changed")); + } + let lease = super::CheckoutLease { + file, + record, + snapshot_pins: Some(pins), + }; + lease.verify_snapshot()?; + Ok(SnapshotClaim { lease }) +} impl CheckoutOwner { fn validate_fields(&self) -> io::Result<()> { super::safe_id(&self.activity_id, "op")?; diff --git a/crates/cyber-core/tests/worktree_repository/activity.rs b/crates/cyber-core/tests/worktree_repository/activity.rs index 5b6424e55..15eb5f752 100644 --- a/crates/cyber-core/tests/worktree_repository/activity.rs +++ b/crates/cyber-core/tests/worktree_repository/activity.rs @@ -1,6 +1,81 @@ use super::*; use cyber_core::worktrees::CheckoutActivity; +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[test] +fn exact_snapshot_reacquisition_preserves_unknown_state_and_refuses_live_foreign_and_replaced_evidence() + { + use cyber_core::worktrees::CheckoutOwner; + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("reacquire-snapshot").unwrap(), + &Settings::default(), + ) + .unwrap(); + let owner = CheckoutOwner { + activity_id: "op_bound".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: managed.path.clone(), + lifetime_digest: "a".repeat(64), + }; + let lease = + block_on(repository.claim_snapshot_bound(&fixture.execution, &managed, &owner)).unwrap(); + let binding = lease.snapshot_binding().unwrap().clone(); + assert!( + repository + .reacquire_snapshot(&managed, &binding, &owner) + .is_err() + ); + drop(lease); // Unknown native outcome remains; reacquisition must not settle it. + let path = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id) + .join("op_bound.lock"); + let original = std::fs::read(&path).unwrap(); + let proof = repository + .reacquire_snapshot(&managed, &binding, &owner) + .unwrap(); + assert_eq!(proof.binding(), &binding); + proof.verify().unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), original); + assert!( + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)) + .is_err() + ); + drop(proof); + assert_eq!(std::fs::read(&path).unwrap(), original); + let mut foreign = owner.clone(); + foreign.session_created_id = "evt_foreign".into(); + assert!( + repository + .reacquire_snapshot(&managed, &binding, &foreign) + .is_err() + ); + let mut legacy = binding.clone(); + legacy.version = 1; + legacy.lifetime = None; + assert!( + repository + .reacquire_snapshot(&managed, &legacy, &owner) + .is_err() + ); + let retained = path.with_extension("retained"); + std::fs::rename(&path, &retained).unwrap(); + std::fs::write(&path, &original).unwrap(); + assert!( + repository + .reacquire_snapshot(&managed, &binding, &owner) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), original); + assert_eq!(std::fs::read(retained).unwrap(), original); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn lifetime_bound_snapshot_claim_publishes_complete_context_without_reusing_existing_owners() { diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index e84f72975..03ab85846 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -551,7 +551,11 @@ impl Scope { // Publish before enqueueing. Even an unavailable writer retains the exact // owner; a later writer acknowledgement can clear it without racing a // post-commit caller-side flag write. - if (self.launched || original.snapshot_request.is_some()) && original.snapshot_worker { + if original.snapshot_worker + && (self.launched + || original.snapshot_request.is_some() + || original.snapshot_lifetime.is_some()) + { handle.mark_snapshot_uncertain(&id); } let (_, uncertain) = self.store.append_checked(&self.id, Expected::Any, move |tx| { @@ -607,13 +611,18 @@ impl Drop for Scope { // possible file effects remain in the separate document intent. let mut disposed = self.snapshot_record(); disposed.snapshot_origin = self.snapshot_origin().cloned(); - disposed.status = - if (self.launched || self.record.snapshot_request.is_some()) && !filesystem_only { - "unknown" - } else { - "settled" - } - .into(); + // A lifetime-bound reservation can already own checkout evidence + // before origin publication; only exact reconciliation clears it. + disposed.status = if (self.launched + || self.record.snapshot_request.is_some() + || self.record.snapshot_worker && self.record.snapshot_lifetime.is_some()) + && !filesystem_only + { + "unknown" + } else { + "settled" + } + .into(); if filesystem_only { disposed.phase = "filesystem_worker_disposed_query_document".into(); } diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs index beaf04bda..5faed82c1 100644 --- a/crates/cyber-server/src/runtime/snapshot_lifetime.rs +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -112,6 +112,20 @@ pub trait SnapshotLifetime: Send { /// Synchronous native metadata verification; may run inside the Store writer. /// It must not call the runtime/Store, launch processes or perform source effects. fn verify(&self) -> Result<(), String>; + /// Publish the complete acquired set before origin/effect dispatch. This is + /// native metadata only; it cannot acknowledge claims or reenter the Store. + fn bind_checkouts( + &self, + _bindings: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result<(), String> { + Err("Durable checkout resource publication is unavailable".into()) + } + /// Missing is not an empty complete resource set and grants no new authority. + fn checkout_bindings( + &self, + ) -> Result>, String> { + Ok(None) + } /// Persist native admission before a source callback. A legacy lifetime may /// never grant the new positive pre-effect cancellation proof. fn admit_source(&self, _origin: &super::SnapshotOrigin) -> Result<(), String> { diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 4e777444b..572a58a5a 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -474,6 +474,11 @@ impl Runtime { } let location = SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; + if let Some(lifetime) = &lifetime { + lifetime + .bind_checkouts(&location.bindings().map_err(RuntimeError::Conflict)?) + .map_err(RuntimeError::Conflict)?; + } Ok(SnapshotWorker { scope, admission, @@ -539,6 +544,11 @@ impl Runtime { let worktree_id = state.info.worktree_id.clone(); let location = SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; + if let Some(lifetime) = &lifetime { + lifetime + .bind_checkouts(&location.bindings().map_err(RuntimeError::Conflict)?) + .map_err(RuntimeError::Conflict)?; + } Ok(SnapshotWorker { scope, runtime: self.clone(), diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 5403a824d..3abd189d3 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -230,6 +230,7 @@ async fn checkout_lifetime_context_is_checked_before_any_native_settlement() { inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, + reject_resources: false, }), ); let owner = worker(&runtime, &session).await; @@ -489,12 +490,27 @@ struct Lifetimes { inner: Acknowledgements, permits: Arc, foreign: bool, + reject_resources: bool, } struct LifetimePermit { binding: super::super::SnapshotLifetimeBinding, _permit: tokio::sync::OwnedSemaphorePermit, + reject_resources: bool, } impl super::super::SnapshotLifetime for LifetimePermit { + fn bind_checkouts( + &self, + bindings: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result<(), String> { + if self.reject_resources { + return Err("injected resource publication failure".into()); + } + if bindings.is_empty() { + Ok(()) + } else { + Err("instrumented lifetime supports unmanaged resource fixtures only".into()) + } + } fn admit_source(&self, _: &SnapshotOrigin) -> Result<(), String> { Ok(()) } @@ -528,6 +544,7 @@ impl Snapshots for Lifetimes { .map_err(|error| error.to_string())?; Ok(Box::new(LifetimePermit { binding, + reject_resources: self.reject_resources, _permit: permit, }) as Box) }) @@ -559,6 +576,7 @@ impl Snapshots for Lifetimes { file_identity: "fixture-file".into(), source_phase: Some("phase_fixture".into()), }, + reject_resources: self.reject_resources, _permit: permit, }) as Box)) @@ -626,6 +644,7 @@ async fn published_origin_with_positive_false_admission_can_cancel_but_admitted_ inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, + reject_resources: false, }), ); let native_binding = binding(temp.path().canonicalize().unwrap()); @@ -682,6 +701,7 @@ async fn rejected_source_admission_prevents_the_callback_and_retains_positive_pr inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, + reject_resources: false, }), ); let owner = worker(&runtime, &session).await; @@ -745,6 +765,7 @@ async fn lost_source_admission_reply_never_calls_the_source_and_never_becomes_fa inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, + reject_resources: false, }), ); let session = runtime @@ -823,6 +844,7 @@ async fn preparation_lost_commit_reply_is_lookup_only_after_reopening_the_real_s inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, + reject_resources: false, }), ); let session = runtime @@ -896,6 +918,7 @@ async fn preparation_cancellation_is_paired_and_retry_never_reopens_native_proof inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: permits.clone(), foreign: false, + reject_resources: false, }), ); let owner = worker(&runtime, &session).await; @@ -985,6 +1008,7 @@ async fn preparation_writer_failure_rolls_back_both_owners_and_changed_session_r inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: permits.clone(), foreign: false, + reject_resources: false, }), ); let id = crash_unstarted(worker(&runtime, &session).await); @@ -1061,6 +1085,7 @@ async fn cancelling_an_unstarted_resolution_keeps_its_original_failed_restore_fe inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, + reject_resources: false, }), ); let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; @@ -1125,6 +1150,7 @@ async fn pre_origin_lifetime_is_bound_before_reservation_and_retained_through_re inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: permits.clone(), foreign: false, + reject_resources: false, }), ); let owner = worker(&runtime, &session).await; @@ -1167,6 +1193,45 @@ async fn pre_origin_lifetime_is_bound_before_reservation_and_retained_through_re base.shutdown().await; } +#[tokio::test] +async fn resource_publication_failure_retains_unknown_reservation_without_origin_or_source_work() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let acknowledgements = Arc::new(AtomicUsize::new(0)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(acknowledgements.clone()), + permits: permits.clone(), + foreign: false, + reject_resources: true, + }), + ); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let error = match runtime.snapshot_worker(&handle, &state).await { + Ok(_) => panic!("worker admitted without durable resource publication"), + Err(error) => error, + }; + assert!( + matches!(error, RuntimeError::Conflict(message) if message.contains("resource publication failure")) + ); + drop(state); + assert_eq!(permits.available_permits(), 1); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); + let activity = latest(&runtime, &session); + assert_eq!(activity["status"], "unknown"); + assert!(activity["snapshot_origin"].is_null()); + assert_ne!(activity["snapshot_source_admitted"], true); + assert!(handle.has_location_uncertainty()); + assert!(!handle.location_uncertain.load(Ordering::SeqCst)); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); + runtime.shutdown().await; + base.shutdown().await; +} + #[tokio::test] async fn foreign_lifetime_proof_is_rejected_before_any_native_reservation() { let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; @@ -1179,6 +1244,7 @@ async fn foreign_lifetime_proof_is_rejected_before_any_native_reservation() { inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), permits: permits.clone(), foreign: true, + reject_resources: false, }), ); let handle = runtime.inner.handle(&session).await.unwrap(); diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index a6bcdf2ec..716944b46 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -15,6 +15,38 @@ struct SourcePhase { lifetime: SnapshotLifetimeBinding, admitted: bool, origin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + checkout_bindings: Option>, +} + +fn validate_checkouts( + binding: &SnapshotLifetimeBinding, + resources: &[cyber_core::worktrees::CheckoutBinding], +) -> Result<(), String> { + let owner = binding.checkout_owner()?; + if resources.len() > 4096 + || binding + .owner + .worktree_id + .as_ref() + .is_some_and(|id| !resources.iter().any(|resource| &resource.worktree_id == id)) + || binding.owner.worktree_id.is_none() && !resources.is_empty() + { + return Err("Snapshot checkout resource set is incomplete or exceeds limits".into()); + } + let mut seen = std::collections::HashSet::new(); + for resource in resources { + if resource.version != 2 + || resource.owner_id != binding.owner.activity_id + || resource.lifetime.as_ref() != Some(&owner) + || !seen.insert((&resource.common_dir, &resource.worktree_id)) + { + return Err( + "Snapshot checkout resource set has foreign ownership or duplicates".into(), + ); + } + } + Ok(()) } pub(crate) struct Lifetime { @@ -30,6 +62,30 @@ impl Drop for Lifetime { } impl SnapshotLifetime for Lifetime { + fn bind_checkouts( + &self, + bindings: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result<(), String> { + self.verify()?; + let store = Store::recover_existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )?; + store.record_checkout_bindings(&self.binding, bindings) + } + fn checkout_bindings( + &self, + ) -> Result>, String> { + self.verify()?; + let store = Store::existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Missing snapshot resource namespace")?; + Ok(store.read_source_phase(&self.binding)?.checkout_bindings) + } fn binding(&self) -> &SnapshotLifetimeBinding { &self.binding } @@ -131,6 +187,51 @@ fn valid_owner(owner: &SnapshotLifetimeRequest) -> Result<(), String> { } impl Store { + fn record_checkout_bindings( + &self, + binding: &SnapshotLifetimeBinding, + resources: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result<(), String> { + validate_checkouts(binding, resources)?; + let mut phase = self.read_source_phase(binding)?; + if phase.admitted { + return Err("Source admission prevents checkout resource publication".into()); + } + if let Some(existing) = &phase.checkout_bindings { + return if existing == resources { + Ok(()) + } else { + Err("Snapshot checkout resource set is immutable".into()) + }; + } + phase.checkout_bindings = Some(resources.to_vec()); + self.write_source_phase(binding, &phase) + } + fn write_source_phase( + &self, + binding: &SnapshotLifetimeBinding, + phase: &SourcePhase, + ) -> Result<(), String> { + let bytes = serde_json::to_vec(phase).map_err(storage::fail)?; + if bytes.len() as u64 > cyber_server::runtime::SnapshotPreparationReceipt::MAX_ENCODED_BYTES + { + return Err("Native source phase exceeds limit".into()); + } + self.reserve( + (bytes.len() as u64) + .checked_mul(2) + .ok_or("Snapshot source size overflow")?, + )?; + let directory = storage::existing_child(&self.namespace, "workers", true)?; + storage::atomic( + &directory, + binding + .source_phase + .as_deref() + .ok_or("Missing source phase")?, + &bytes, + ) + } pub(crate) fn reserve_lifetime( &self, owner: SnapshotLifetimeRequest, @@ -170,6 +271,7 @@ impl Store { lifetime: binding.clone(), admitted: false, origin: None, + checkout_bindings: None, }) .map_err(storage::fail)? .len() as u64; @@ -194,6 +296,7 @@ impl Store { lifetime: binding.clone(), admitted: false, origin: None, + checkout_bindings: None, }; storage::create( &directory, @@ -307,6 +410,9 @@ impl Store { if phase.lifetime != *binding || phase.admitted != phase.origin.is_some() { return Err("Native source phase changed owner".into()); } + if let Some(resources) = &phase.checkout_bindings { + validate_checkouts(binding, resources)?; + } Ok(phase) } pub(super) fn verify_native_preparation( @@ -324,6 +430,9 @@ impl Store { origin: &cyber_server::runtime::SnapshotOrigin, ) -> Result<(), String> { let phase = self.read_source_phase(binding)?; + if binding.owner.worktree_id.is_some() && phase.checkout_bindings.is_none() { + return Err("Managed source admission lacks a complete resource set".into()); + } if origin.activity_id != binding.owner.activity_id || origin.session_id != binding.owner.session_id || origin.session_created_id != binding.owner.session_created_id @@ -347,26 +456,9 @@ impl Store { lifetime: binding.clone(), admitted: true, origin: Some(origin.clone()), + checkout_bindings: phase.checkout_bindings, }; - let bytes = serde_json::to_vec(&phase).map_err(storage::fail)?; - if bytes.len() as u64 > cyber_server::runtime::SnapshotPreparationReceipt::MAX_ENCODED_BYTES - { - return Err("Native source phase exceeds limit".into()); - } - self.reserve( - (bytes.len() as u64) - .checked_mul(2) - .ok_or("Snapshot source size overflow")?, - )?; - let directory = storage::existing_child(&self.namespace, "workers", true)?; - storage::atomic( - &directory, - binding - .source_phase - .as_deref() - .ok_or("Missing source phase")?, - &bytes, - ) + self.write_source_phase(binding, &phase) } } @@ -400,6 +492,115 @@ mod tests { .unwrap() } + #[test] + fn complete_resource_publication_is_immutable_quota_checked_and_retained_after_source_admission() + { + let temp = tempfile::tempdir().unwrap(); + let mut store = store(temp.path()); + let mut owner = request(&store.root); + owner.worktree_id = Some("wt_fixture".into()); + let lifetime = store.reserve_lifetime(owner).unwrap(); + let binding = lifetime.binding().clone(); + assert_eq!(lifetime.checkout_bindings().unwrap(), None); + // The core/runtime suites verify actual resource locks. This fixture + // exercises the native reference-publication state and durability. + let resource = cyber_core::worktrees::CheckoutBinding { + version: 2, + owner_id: binding.owner.activity_id.clone(), + worktree_id: "wt_fixture".into(), + name: "fixture".into(), + common_dir: store.root.clone(), + common_identity: "fixture".into(), + root_identity: "fixture".into(), + directory_identity: "fixture".into(), + file_identity: "fixture".into(), + creation_directory_identity: "fixture".into(), + creation_file_identity: "fixture".into(), + creation_digest: "a".repeat(64), + lifetime: Some(binding.checkout_owner().unwrap()), + staging_name: Some("fixture".into()), + }; + let mut foreign = resource.clone(); + foreign.lifetime.as_mut().unwrap().session_created_id = "evt_foreign".into(); + assert!( + store + .record_checkout_bindings(&binding, &[foreign]) + .is_err() + ); + assert!(store.record_checkout_bindings(&binding, &[]).is_err()); + assert!( + store + .record_checkout_bindings(&binding, &[resource.clone(), resource.clone()]) + .is_err() + ); + assert_eq!(lifetime.checkout_bindings().unwrap(), None); + store.options.max_total = 1; + assert!( + store + .record_checkout_bindings(&binding, std::slice::from_ref(&resource)) + .is_err() + ); + assert_eq!(lifetime.checkout_bindings().unwrap(), None); + store.options.max_total = 4 * 1024 * 1024; + let id = "restore_resource_test"; + let directory = storage::private_child(&store.journals, id).unwrap(); + let origin = cyber_server::runtime::SnapshotOrigin { + version: 1, + activity_id: binding.owner.activity_id.clone(), + session_id: binding.owner.session_id.clone(), + session_created_id: binding.owner.session_created_id.clone(), + worktree_id: binding.owner.worktree_id.clone(), + binding: cyber_server::runtime::SnapshotBinding { + directory: store.root.clone(), + root_identity: store.root_id.clone(), + namespace_identity: storage::dir_identity(&store.journals).unwrap(), + journal_id: id.into(), + journal_identity: storage::dir_identity(&directory).unwrap(), + lock_identity: Some(store.lock_identity().unwrap()), + }, + }; + assert!(lifetime.admit_source(&origin).is_err()); + let resources = vec![resource]; + store + .record_checkout_bindings(&binding, &resources) + .unwrap(); + store + .record_checkout_bindings(&binding, &resources) + .unwrap(); + let mut changed = resources.clone(); + changed[0].file_identity.push_str("_replacement"); + assert!(store.record_checkout_bindings(&binding, &changed).is_err()); + assert_eq!( + lifetime.checkout_bindings().unwrap(), + Some(resources.clone()) + ); + let mut journal = store.prepare_journal(vec![], id, None, None, None).unwrap(); + journal.origin = Some(origin.clone()); + journal.ownership = + Some(super::super::ownership::Ownership::prepare(&store, &directory, &[]).unwrap()); + Store::record(&directory, &journal).unwrap(); + lifetime.admit_source(&origin).unwrap(); + assert!( + store + .record_checkout_bindings(&binding, &resources) + .is_err() + ); + drop(store); + assert_eq!(lifetime.checkout_bindings().unwrap(), Some(resources)); + } + + #[test] + fn empty_complete_resource_set_is_distinct_from_missing() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let lifetime = store.reserve_lifetime(request(&store.root)).unwrap(); + assert_eq!(lifetime.checkout_bindings().unwrap(), None); + drop(store); + lifetime.bind_checkouts(&[]).unwrap(); + assert_eq!(lifetime.checkout_bindings().unwrap(), Some(vec![])); + lifetime.bind_checkouts(&[]).unwrap(); + } + #[test] fn exact_lifetime_lock_retains_privacy_quota_and_rejects_replacement_or_missing_proof() { use std::os::unix::fs::PermissionsExt; diff --git a/crates/cyber-store/migrations/20261107000000_snapshot_checkout_resources.sql b/crates/cyber-store/migrations/20261107000000_snapshot_checkout_resources.sql new file mode 100644 index 000000000..ba215ba41 --- /dev/null +++ b/crates/cyber-store/migrations/20261107000000_snapshot_checkout_resources.sql @@ -0,0 +1,3 @@ +-- Complete native resource sets precede source admission. Older writers cannot +-- bypass reference publication or erase the new native phase fields. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index bb1564057..01f50f701 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -189,6 +189,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261106030000_snapshot_checkout_lifetime", sql: include_str!("../migrations/20261106030000_snapshot_checkout_lifetime.sql"), }, + Migration { + id: "20261107000000_snapshot_checkout_resources", + sql: include_str!("../migrations/20261107000000_snapshot_checkout_resources.sql"), + }, ]; pub(crate) enum Outcome { @@ -436,7 +440,13 @@ mod tests { let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { panic!("activity-only writer accepted lifetime-bound checkout storage"); }; - assert_eq!(unknown, ["20261106030000_snapshot_checkout_lifetime"]); + assert_eq!( + unknown, + MIGRATIONS[at..] + .iter() + .map(|migration| migration.id) + .collect::>() + ); let retained: String = db .query_row("SELECT data FROM event WHERE id='evt_context'", [], |row| { row.get(0) @@ -445,6 +455,35 @@ mod tests { assert_eq!(retained, evidence); } + #[test] + fn native_resource_catalog_blocks_previous_writers_without_changing_reference_evidence() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("resources.db"); + let phase = directory.path().join("phase_retained"); + let evidence = br#"{"admitted":false,"checkout_bindings":[]}"#; + std::fs::write(&phase, evidence).unwrap(); + let at = MIGRATIONS + .iter() + .position(|migration| migration.id == "20261107000000_snapshot_checkout_resources") + .unwrap(); + let mut db = Connection::open(&path).unwrap(); + assert!(matches!( + migrate(&mut db, &MIGRATIONS[..at]).unwrap(), + Outcome::Current + )); + assert!(matches!( + migrate(&mut db, MIGRATIONS).unwrap(), + Outcome::Current + )); + drop(db); + let mut db = Connection::open(path).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { + panic!("previous writer accepted native resource publication"); + }; + assert_eq!(unknown, ["20261107000000_snapshot_checkout_resources"]); + assert_eq!(std::fs::read(phase).unwrap(), evidence); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index c98a68c33..830bee7f1 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -179,3 +179,11 @@ native settlement; prior writer catalogs refuse mutation. Construction kills ret evidence and refuse replay. Unknown-claim acknowledgement and reclamation remain incomplete, and Windows lifetime-bound durability is not yet admitted. This does not accept the complete recovery feature or either issue. + + +The local acquired-resource continuation adds exact read-only exited-claim +reacquisition and durable complete checkout references before source dispatch. +Publication failure preserves the unknown reservation; no new native acknowledgement +or cleanup authority is granted. Interrupted-claim consuming receipts and native +reconciliation remain the next recovery work, alongside the existing Git, +reclamation and platform/client acceptance obligations. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 52105b751..f723c46cf 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1389,3 +1389,77 @@ findings remain for initialized lifetime-bound publication. Full unknown-resourc reconciliation, complete ownership/reference publication across all cuts, Git recovery, reclamation and platform/client acceptance remain open. This local continuation does not make the complete recovery increment merge-ready or complete either issue. + + +## Exact exited-claim reacquisition and durable acquired resources (2026-11-07) + +Core exposes a separate read-only `SnapshotClaim`: callers supply the exact +version-2 binding, immutable lifetime context and managed creation. Reacquisition +opens existing evidence, takes the native exclusive lock and verifies retained +common/activity/creation pins, creation bytes, file identity and complete record. +It does not create or rewrite a claim and exposes no settlement API. Real managed +checkout coverage checks live-worker refusal, unchanged unknown bytes, retained +removal exclusion, foreign creation/lifetime and legacy refusal, and a byte-identical +replacement refusal while retaining both original and replacement evidence. + +Ordinary and recovery runtime factories publish their verified complete acquired +binding set through the lifetime port before returning a worker. The private source +phase retains `Some([])` for a complete unmanaged acquisition and `None` for absent +constructor evidence. Publication holds the native global snapshot lock for quota +accounting, checks bounded ownership/context and duplicates, refuses changed sets +or publication after source admission, and retains the original references when +admitting source work. Managed source admission refuses absent resource evidence. +The compatibility catalog marker prevents previous Store writers from erasing the +new phase fields; its migration regression preserves retained reference bytes. + +Native tests cover missing/empty, foreign/duplicate, quota failure, identical retry, +immutable replacement and post-admission reference retention. The resource DTOs in +that state-machine fixture are instrumented; actual file/lock ownership is covered +by the core integration and existing production-host tests. Runtime failure injection +checks that rejected publication returns no worker, retains an unknown reservation +with no origin/source admission, releases admission/lifetime permits and performs +no acknowledgement. Real CLI successful restore assertions inspect the actual +private source phase and require its original lifetime/origin and explicit empty +resource set in unmanaged fixtures. + +The code-review skill was applied across every changed core/runtime/native/Store, +test and spec file. Review checks no-follow existing-only reacquisition, exact +creation/record comparison, retained locks on all exits, no generic settlement on +read-only proof, private bounded atomic publication under native quota ownership, +resource immutability and source ordering, and backward writer refusal. The added +runtime failure and managed native missing-set regressions close the observed test +gaps. This step enables no unknown-claim acknowledgement and does not complete +partial-constructor reconciliation, Git recovery, reference-aware reclamation, +M2–M8 or either issue. The final runtime regression found that pre-origin disposal could settle a lifetime-bound +reservation after failed resource publication. Disposal now preserves that exact +snapshot uncertainty until explicit reconciliation; generic and binary scopes keep +their existing disposal rules. + + +Verification for this continuation on local macOS arm64: the broad runs passed +84 snapshot cases (one ignored child fixture), 423 server unit cases (one ignored +fixture) plus 22 affected server integrations, 62 Store cases and 72 core worktree +integration cases (one ignored child fixture). After final creation validation, +the exact native core reacquisition regression passed again. All 16 real CLI +branch/recovery cases passed with the new durable resource assertions. All-target +Clippy for core/server/Store/snapshot/tools/App/CLI passed with warnings denied. +Strict pinned OpenSpec passed 117 items, spec lint reports 0 errors/25 warnings, +and generated SDK consistency, formatting and diff checks passed. Native sandbox +API deprecation warnings were observed; no unchanged-main comparison or origin +attribution is made here. Full-workspace tests, SDK execution tests, native Linux/ +Windows runs and live TUI/App recovery acceptance were not run for this increment. + + +The final native missing-managed-resource admission regression passed. The initial +focused runtime run passed 40 cases and failed the new publication-failure case, +exposing the pre-origin disposal gap above. After the fix all 424 server unit +cases passed with one ignored fixture, and server all-target Clippy passed again. +The first test-selection command matched zero tests and was replaced by the actual +snapshot module selection; no coverage is attributed to that empty run. + +The final focused disposal regression passed with explicit exact snapshot +uncertainty and no opaque Location flag. All 16 real CLI cases passed again after +the disposal fix, including both pre-origin/pre-source service kills and partial +keep/rollback restart cases. Final formatting/diff checks passed. Review found no +remaining critical/major issue within this increment; the broader recovery and +platform/client acceptance gaps above remain required before delivery. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 050f7a4f6..0535ecaa9 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -368,3 +368,23 @@ publication, while a retained extra link refuses verification. It does not gain lifetime-bound durability or recovery authority; that path refuses before creating claim evidence until native directory durability has been implemented and verified. This remains part of the explicit Windows/platform acceptance limits. + + +### Reacquisition and durable acquired-resource publication + +An exited claim may be reacquired only from its exact version-2 binding and matching +immutable lifetime context. Reacquisition opens existing native evidence without +creating a name, resetting settled state or acknowledging a Session/process claim. +It retains the original lock and metadata pins; live, missing, replaced, aliased, +legacy or foreign evidence refuses. The read-only proof is distinct from an ordinary +claim and cannot be passed to generic settlement. + +Before a ready worker can publish an origin or dispatch source effects, its actual +complete acquired binding set is published under the private lifetime phase. Native +publication is bounded/quota-checked, cannot replace a different set, and cannot add +references after source admission. Empty complete sets remain distinct from absent +or incomplete publication. Managed source admission requires complete publication. +This resource set is retained alongside source admission/origin and remains protected +for later consuming receipt reconciliation. The current step does not acknowledge +unknown claims or infer completeness from missing files; pre-publication construction +cuts and complete reference/cleanup acceptance remain required. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index e9aa933ff..3659d5e6f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -105,6 +105,17 @@ - **AND** runtime admission SHALL refuse returned resources with foreign or missing lifetime context - **AND** a construction crash SHALL preserve private evidence without granting an incomplete stable record or legacy claim new acknowledgement authority +#### Scenario: Exact exited claim reacquisition is read-only +- **WHEN** recovery reacquires an exited checkout claim under the exact stored version-2 binding and lifetime context +- **THEN** it SHALL retain the existing native lock and metadata pins without creating or resetting any claim or acknowledgement +- **AND** live, aliased, replaced, missing, legacy or foreign evidence SHALL refuse + +#### Scenario: Acquired resource set precedes source admission +- **WHEN** a lifetime-owned worker completes native checkout acquisition +- **THEN** its complete acquired binding set SHALL be durably published before an origin or source callback, with empty and missing sets distinguished +- **AND** a changed set or post-admission addition SHALL refuse, source admission SHALL retain the published set, and incomplete evidence SHALL NOT gain reference-completeness authority +- **AND** disposal before origin publication SHALL retain a lifetime-bound reservation as unknown until explicit exact-owner reconciliation + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 5782c9dab..c2d3999f9 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -210,3 +210,15 @@ collision tests retain evidence and refuse activity reuse. Unbound legacy claims and Windows durability limits remain distinct. Unknown-claim acknowledgement, full publication/recovery cuts, Git journals and reclamation remain incomplete; no parent M2–M8 or 2.4/3.2–3.4 task is accepted here. + + +Read-only reacquisition and acquired-resource publication continuation (2026-11-07): +core now reacquires an exact exited version-2 checkout claim under its native lock, +retaining original metadata pins without creating, settling or changing evidence. +A distinct proof type exposes no generic settlement. Runtime publishes the actual +complete acquired binding set into the private lifetime source phase before origin +or source dispatch. Publication is quota-checked and immutable; source admission +retains references and refuses missing managed sets. Explicit empty unmanaged sets +remain distinct from absent constructor evidence. Unknown-claim acknowledgement, +partial acquisition/publication cuts, Git journals, reclamation and full native/client +acceptance remain open; no parent task is marked complete. From e81192e5a0b0a68d1ee8f5a60ddc527c3017f9cd Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 00:58:46 -0400 Subject: [PATCH 17/87] Bind preparation receipts to retained checkout resources --- crates/cyber-app/src/host.rs | 10 + crates/cyber-cli/tests/branches_cli.rs | 17 ++ crates/cyber-server/src/runtime/host.rs | 17 ++ .../src/runtime/snapshot_lifetime.rs | 46 ++++ .../src/runtime/snapshot_preparation.rs | 86 +++++++- .../cyber-server/src/runtime/snapshot_work.rs | 1 + .../src/runtime/snapshot_work_tests.rs | 198 +++++++++++++++++- crates/cyber-snapshot/src/copy/lifetime.rs | 48 ++--- crates/cyber-snapshot/src/copy/preparation.rs | 46 ++++ ...1107010000_snapshot_consumed_resources.sql | 3 + crates/cyber-store/src/migrate.rs | 37 +++- crates/cyber-tools/src/host.rs | 9 + crates/cyber-tools/src/worktrees.rs | 45 ++++ crates/cyber-tools/tests/worktree_setup.rs | 55 ++++- docs/goal-delivery.md | 9 + .../implementation.md | 83 ++++++++ .../managed-recovery.md | 14 ++ .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 12 ++ sdk/openapi.json | 12 ++ sdk/typescript/src/generated/types.ts | 2 + 21 files changed, 725 insertions(+), 31 deletions(-) create mode 100644 crates/cyber-store/migrations/20261107010000_snapshot_consumed_resources.sql diff --git a/crates/cyber-app/src/host.rs b/crates/cyber-app/src/host.rs index 983713f8b..47d39d8fc 100644 --- a/crates/cyber-app/src/host.rs +++ b/crates/cyber-app/src/host.rs @@ -290,6 +290,16 @@ impl ToolHost for AppHost { .claim_snapshot_location_bound(info, activity, lifetime, cancel) } + fn reacquire_snapshot_location( + &self, + info: &cyber_server::runtime::SessionInfo, + lifetime: &cyber_server::runtime::SnapshotLifetimeBinding, + bindings: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result, String> { + self.builtin + .reacquire_snapshot_location(info, lifetime, bindings) + } + fn prepare_child_continuation<'a>( &'a self, parent: &'a cyber_server::runtime::SessionInfo, diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index cfbea2900..fb36e271a 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -841,6 +841,7 @@ fn cancelled_pre_effect_preparation(published: bool) { let receipt = json(&env.cyber(&["api", "POST", &format!("{route}/confirm"), "-d", &body]))["data"] .clone(); + assert_preparation_resource_digest(&review, &receipt, published); let acknowledge = format!("{route}/requests/before-origin/acknowledge"); assert_eq!( json(&env.cyber(&["api", "POST", &acknowledge]))["data"], @@ -878,6 +879,22 @@ fn cancelled_pre_effect_preparation(published: bool) { env.exec(Some(&id), "after preparation cancellation"); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_preparation_resource_digest(review: &Value, receipt: &Value, published: bool) { + assert_eq!( + receipt["checkout_resources_digest"], + review["checkout_resources_digest"] + ); + if published { + let lifetime: cyber_server::runtime::SnapshotLifetimeBinding = + serde_json::from_value(review["lifetime"].clone()).unwrap(); + assert_eq!( + receipt["checkout_resources_digest"], + lifetime.checkout_resources_digest(&[]).unwrap() + ); + } +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_preparation_tombstone(env: &Env, receipt: &Value) { let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index c0537c798..4be4b4ee0 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -930,6 +930,23 @@ pub trait ToolHost: Send + Sync { } } + /// Reacquire only original snapshot evidence, without creating or settling claims. + /// The caller retains the matching lifetime lock independently. + fn reacquire_snapshot_location( + &self, + info: &SessionInfo, + _lifetime: &super::SnapshotLifetimeBinding, + bindings: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result, String> { + if info.worktree_id.is_none() && bindings.is_empty() { + Ok(Box::new(UnmanagedSnapshotLocation { + _owner: Box::new(()), + })) + } else { + Err("Exact snapshot checkout reacquisition is unsupported".into()) + } + } + /// Prepare an existing child under its exclusive owner. Opening a client view grants no bypass. fn prepare_child_continuation<'a>( &'a self, diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs index 5faed82c1..693f1d42b 100644 --- a/crates/cyber-server/src/runtime/snapshot_lifetime.rs +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -26,6 +26,40 @@ pub struct SnapshotLifetimeBinding { } impl SnapshotLifetimeBinding { + /// Versioned digest of the complete ordered acquisition, bound to this lifetime. + pub fn checkout_resources_digest( + &self, + resources: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result { + use sha2::{Digest, Sha256}; + let owner = self.checkout_owner()?; + if resources.len() > 4096 + || self + .owner + .worktree_id + .as_ref() + .is_some_and(|id| !resources.iter().any(|resource| &resource.worktree_id == id)) + || self.owner.worktree_id.is_none() && !resources.is_empty() + { + return Err("Snapshot checkout resource set is incomplete or exceeds limits".into()); + } + let mut seen = std::collections::HashSet::new(); + for resource in resources { + if resource.version != 2 + || resource.owner_id != self.owner.activity_id + || resource.lifetime.as_ref() != Some(&owner) + || !seen.insert((&resource.common_dir, &resource.worktree_id)) + { + return Err( + "Snapshot checkout resource set has foreign ownership or duplicates".into(), + ); + } + } + let bytes = serde_json::to_vec(&("snapshot-checkout-resources-v1", self, resources)) + .map_err(|error| error.to_string())?; + Ok(format!("{:x}", Sha256::digest(bytes))) + } + pub fn checkout_owner(&self) -> Result { use sha2::{Digest, Sha256}; if self.version != 2 || self.source_phase.is_none() { @@ -172,5 +206,17 @@ mod checkout_encoding_tests { binding.checkout_owner().unwrap().lifetime_digest, "66239118d7327f0739d4e468da0b7d4c547e0a752cfd9680e6e5f7f2469af802" ); + assert_eq!( + binding.checkout_resources_digest(&[]).unwrap(), + "a36ba5baced567a6dde73bc41c6e83c49017ba049b43fd59a5607b6204b5c5e7" + ); + let mut changed = binding.clone(); + changed.file_identity.push_str("_replacement"); + assert_ne!( + changed.checkout_resources_digest(&[]).unwrap(), + binding.checkout_resources_digest(&[]).unwrap() + ); + changed.version = 1; + assert!(changed.checkout_resources_digest(&[]).is_err()); } } diff --git a/crates/cyber-server/src/runtime/snapshot_preparation.rs b/crates/cyber-server/src/runtime/snapshot_preparation.rs index 65c3efbad..864afd92b 100644 --- a/crates/cyber-server/src/runtime/snapshot_preparation.rs +++ b/crates/cyber-server/src/runtime/snapshot_preparation.rs @@ -30,6 +30,8 @@ pub struct SnapshotPreparationReview { pub confirmation: String, pub warning: String, pub origin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkout_resources_digest: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] @@ -44,6 +46,8 @@ pub struct SnapshotPreparationReceipt { pub origin: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub source_admitted: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkout_resources_digest: Option, } impl SnapshotPreparationReceipt { @@ -75,6 +79,15 @@ pub(super) fn validate_event(data: &serde_json::Value) -> Result<(), String> { } fn validate_receipt(receipt: &SnapshotPreparationReceipt) -> Result<(), StoreError> { if !(0..=2).contains(&receipt.native_sequence) + || receipt + .checkout_resources_digest + .as_ref() + .is_some_and(|digest| { + receipt.lifetime.version != 2 + || receipt.lifetime.source_phase.is_none() + || digest.len() != 64 + || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) || receipt.session_sequence < 0 || !matches!(receipt.lifetime.version, 1 | 2) || receipt.lifetime.owner.activity_id != receipt.activity_id @@ -313,20 +326,62 @@ struct Worker { sequence: i64, target: Target, control: Arc, + resources: Option>, + checkouts: Option>, proof: Box, _lifecycle: tokio::sync::OwnedRwLockReadGuard<()>, _admission: Admission, _completion: Completion, } impl Worker { + fn resources_digest(&self) -> Result, RuntimeError> { + self.resources + .as_ref() + .map(|resources| { + self.target + .binding + .checkout_resources_digest(resources) + .map_err(RuntimeError::Conflict) + }) + .transpose() + } + fn verify_resources(&self) -> Result<(), String> { + if self.proof.checkout_bindings()? != self.resources { + return Err("Snapshot preparation resource publication changed".into()); + } + match (&self.resources, &self.checkouts) { + (Some(resources), Some(checkouts)) => { + checkouts.verify()?; + if checkouts.bindings()? != *resources { + return Err("Retained snapshot preparation resources changed".into()); + } + } + (None, None) => {} + _ => return Err("Snapshot preparation resource proof is incomplete".into()), + } + if let Some(expected) = self + .target + .record + .snapshot_preparation_receipt + .as_ref() + .and_then(|receipt| receipt.checkout_resources_digest.as_ref()) + { + let observed = self.resources_digest().map_err(|error| error.to_string())?; + if observed.as_ref() != Some(expected) { + return Err("Committed preparation resource binding differs".into()); + } + } + Ok(()) + } fn fingerprint(&self) -> Result { use sha2::{Digest, Sha256}; let bytes = serde_json::to_vec(&( - "snapshot-preparation-review-v1", + "snapshot-preparation-review-v2", self.sequence, &self.authority, self.target.sequence, &self.target.raw, + self.resources_digest()?, )) .map_err(|error| RuntimeError::Invalid(error.to_string()))?; Ok(format!("{:x}", Sha256::digest(bytes))) @@ -338,6 +393,7 @@ impl Worker { self.proof .verify_preparation(self.target.record.snapshot_origin.as_ref()) .map_err(|error| invalid(&error))?; + self.verify_resources().map_err(|error| invalid(&error))?; let owner = &self.target.binding.owner; let current = match self.target.record.snapshot_preparation_receipt.as_ref() { Some(receipt) => acknowledged_target(db, receipt)?, @@ -352,6 +408,7 @@ impl Worker { || current.sequence != self.target.sequence || sequence != self.sequence || !self.authority.current(db)? + || self.control.stop.is_cancelled() { return Err(invalid("Snapshot preparation authority or owner changed")); } @@ -377,6 +434,7 @@ impl Worker { session_sequence: self.sequence, origin: self.target.record.snapshot_origin.clone(), source_admitted: self.target.record.snapshot_source_admitted, + checkout_resources_digest: self.resources_digest()?, }; let mut record = self.target.record.clone(); record.status = "settled".into(); @@ -497,6 +555,7 @@ impl Runtime { } let authority = RecoveryAuthority::capture(self, session)?; let sequence = state.last_seq; + let info = state.info.clone(); handle .snapshot_active .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) @@ -534,18 +593,36 @@ impl Runtime { "Foreign preparation lifetime proof".into(), )); } - Ok(Worker { + let resources = proof.checkout_bindings().map_err(RuntimeError::Conflict)?; + let checkouts = resources + .as_ref() + .map(|resources| { + target + .binding + .checkout_resources_digest(resources) + .map_err(RuntimeError::Conflict)?; + self.inner + .tools + .reacquire_snapshot_location(&info, &target.binding, resources) + .map_err(RuntimeError::Conflict) + }) + .transpose()?; + let worker = Worker { runtime: self.clone(), handle, authority, sequence, target, control, + resources, + checkouts, proof, _lifecycle: lifecycle, _admission: admission, _completion: completion, - }) + }; + worker.verify_resources().map_err(RuntimeError::Conflict)?; + Ok(worker) } pub async fn snapshot_preparation_review( &self, @@ -558,6 +635,7 @@ impl Runtime { let worker = store.read(move |db| { let tx = db.unchecked_transaction()?; worker.authorize(&tx)?; tx.commit()?; Ok(worker) }).map_err(preparation_error)?; Ok(SnapshotPreparationReview { activity_id: worker.target.binding.owner.activity_id.clone(), lifetime: worker.target.binding.clone(), native_sequence: worker.target.sequence, session_sequence: worker.sequence, confirmation: worker.fingerprint()?, + checkout_resources_digest: worker.resources_digest()?, origin: worker.target.record.snapshot_origin.clone(), warning: "Cancels only this pre-effect snapshot preparation. Files and transcript are preserved; checkout and process uncertainty remain independent.".into() }) }).await.map_err(|_| RuntimeError::Conflict("Snapshot preparation review interrupted".into()))? @@ -607,7 +685,7 @@ impl Runtime { .ok_or_else(|| { RuntimeError::Conflict("No committed preparation cancellation receipt".into()) })?; - if receipt.origin.is_none() { + if receipt.origin.is_none() && receipt.checkout_resources_digest.is_none() { return Ok(receipt); } let worker = self diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 572a58a5a..d12696f1f 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -134,6 +134,7 @@ impl SnapshotWork<'_> { session_sequence: i64::MAX, origin: Some(origin.clone()), source_admitted: Some(false), + checkout_resources_digest: Some("0".repeat(64)), }; let bytes = serde_json::to_vec(&receipt) .map_err(|error| error.to_string())? diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 3abd189d3..e787455f1 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -71,11 +71,20 @@ fn runtime_with_snapshots( store: Arc, root: &std::path::Path, snapshots: Arc, +) -> Runtime { + runtime_with_snapshot_tools(runtime, store, root, snapshots, runtime.inner.tools.clone()) +} +fn runtime_with_snapshot_tools( + runtime: &Runtime, + store: Arc, + root: &std::path::Path, + snapshots: Arc, + tools: Arc, ) -> Runtime { Runtime::new(RuntimeOptions { store, resolver: runtime.inner.resolver.clone(), - tools: runtime.inner.tools.clone(), + tools, global_config_dir: root.join("config"), shell: "test".into(), claude_compat: false, @@ -92,6 +101,9 @@ fn runtime_with_snapshots( struct SnapshotClaims { ordinary_forbidden: std::sync::atomic::AtomicBool, claimed: std::sync::Mutex>, + reacquisition: Option<(Arc, Arc)>, + reacquire_stop: bool, + reacquire_inner: std::sync::Mutex>, } impl super::super::ToolHost for SnapshotClaims { fn definitions(&self, _: &super::super::TurnContext) -> Vec { @@ -104,6 +116,38 @@ impl super::super::ToolHost for SnapshotClaims { ) -> futures::future::BoxFuture<'_, super::super::ToolOutcome> { Box::pin(async { super::super::ToolOutcome::Failed("unused fixture tool".into()) }) } + fn reacquire_snapshot_location( + &self, + info: &super::super::SessionInfo, + _: &super::super::SnapshotLifetimeBinding, + _: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result, String> { + let (calls, fail_at) = self + .reacquisition + .as_ref() + .ok_or("no reacquisition fixture")?; + if self.reacquire_stop { + let inner = self.reacquire_inner.lock().unwrap().upgrade().unwrap(); + let stop = inner + .activities + .lock() + .unwrap() + .values() + .find(|control| control.source == info.id && control.snapshot_worker) + .unwrap() + .stop + .clone(); + return Ok(Box::new(StopCheckoutProof { + calls: calls.clone(), + stop_at: fail_at.clone(), + stop, + })); + } + Ok(Box::new(CheckoutProof { + calls: calls.clone(), + fail_at: fail_at.clone(), + })) + } fn claim_location<'a>( &'a self, _: &'a super::super::SessionInfo, @@ -231,6 +275,7 @@ async fn checkout_lifetime_context_is_checked_before_any_native_settlement() { permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let owner = worker(&runtime, &session).await; @@ -491,11 +536,13 @@ struct Lifetimes { permits: Arc, foreign: bool, reject_resources: bool, + resources: Arc>>>, } struct LifetimePermit { binding: super::super::SnapshotLifetimeBinding, _permit: tokio::sync::OwnedSemaphorePermit, reject_resources: bool, + resources: Arc>>>, } impl super::super::SnapshotLifetime for LifetimePermit { fn bind_checkouts( @@ -506,11 +553,17 @@ impl super::super::SnapshotLifetime for LifetimePermit { return Err("injected resource publication failure".into()); } if bindings.is_empty() { + *self.resources.lock().unwrap() = Some(vec![]); Ok(()) } else { Err("instrumented lifetime supports unmanaged resource fixtures only".into()) } } + fn checkout_bindings( + &self, + ) -> Result>, String> { + Ok(self.resources.lock().unwrap().clone()) + } fn admit_source(&self, _: &SnapshotOrigin) -> Result<(), String> { Ok(()) } @@ -545,6 +598,7 @@ impl Snapshots for Lifetimes { Ok(Box::new(LifetimePermit { binding, reject_resources: self.reject_resources, + resources: self.resources.clone(), _permit: permit, }) as Box) }) @@ -577,6 +631,7 @@ impl Snapshots for Lifetimes { source_phase: Some("phase_fixture".into()), }, reject_resources: self.reject_resources, + resources: self.resources.clone(), _permit: permit, }) as Box)) @@ -645,6 +700,7 @@ async fn published_origin_with_positive_false_admission_can_cancel_but_admitted_ permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let native_binding = binding(temp.path().canonicalize().unwrap()); @@ -702,6 +758,7 @@ async fn rejected_source_admission_prevents_the_callback_and_retains_positive_pr permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let owner = worker(&runtime, &session).await; @@ -766,6 +823,7 @@ async fn lost_source_admission_reply_never_calls_the_source_and_never_becomes_fa permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let session = runtime @@ -845,6 +903,7 @@ async fn preparation_lost_commit_reply_is_lookup_only_after_reopening_the_real_s permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let session = runtime @@ -919,6 +978,7 @@ async fn preparation_cancellation_is_paired_and_retry_never_reopens_native_proof permits: permits.clone(), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let owner = worker(&runtime, &session).await; @@ -1009,6 +1069,7 @@ async fn preparation_writer_failure_rolls_back_both_owners_and_changed_session_r permits: permits.clone(), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let id = crash_unstarted(worker(&runtime, &session).await); @@ -1086,6 +1147,7 @@ async fn cancelling_an_unstarted_resolution_keeps_its_original_failed_restore_fe permits: Arc::new(tokio::sync::Semaphore::new(1)), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let original = failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; @@ -1151,6 +1213,7 @@ async fn pre_origin_lifetime_is_bound_before_reservation_and_retained_through_re permits: permits.clone(), foreign: false, reject_resources: false, + resources: Default::default(), }), ); let owner = worker(&runtime, &session).await; @@ -1193,6 +1256,137 @@ async fn pre_origin_lifetime_is_bound_before_reservation_and_retained_through_re base.shutdown().await; } +#[tokio::test] +async fn preparation_consuming_writer_revalidates_original_checkout_proof_and_rolls_back_both_owners() + { + for stop in [false, true] { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let calls = Arc::new(AtomicUsize::new(0)); + let fail_at = Arc::new(AtomicUsize::new(usize::MAX)); + let host = Arc::new(SnapshotClaims { + reacquisition: Some((calls.clone(), fail_at.clone())), + reacquire_stop: stop, + ..Default::default() + }); + let runtime = runtime_with_snapshot_tools( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + reject_resources: false, + resources: Default::default(), + }), + host.clone(), + ); + *host.reacquire_inner.lock().unwrap() = Arc::downgrade(&runtime.inner); + let id = crash_unstarted(worker(&runtime, &session).await); + let review = runtime + .snapshot_preparation_review(&session, &id) + .await + .unwrap(); + let before = calls.load(Ordering::SeqCst); + fail_at.store(before + 2, Ordering::SeqCst); // factory succeeds; consuming writer refuses. + let confirmation = super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "checkout-writer-proof".into(), + }; + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .is_err() + ); + assert_eq!(calls.load(Ordering::SeqCst), before + 2); + assert!( + runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap() + .is_none() + ); + assert_ne!(latest(&runtime, &session)["status"], "settled"); + fail_at.store(usize::MAX, Ordering::SeqCst); + let receipt = runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .unwrap(); + assert_eq!( + receipt.checkout_resources_digest, + review.checkout_resources_digest + ); + runtime.shutdown().await; + base.shutdown().await; + } +} + +#[tokio::test] +async fn preparation_receipt_binds_resources_and_lookup_never_reopens_changed_native_evidence() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let resources = Arc::new(std::sync::Mutex::new(None)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + reject_resources: false, + resources: resources.clone(), + }), + ); + let id = crash_unstarted(worker(&runtime, &session).await); + let review = runtime + .snapshot_preparation_review(&session, &id) + .await + .unwrap(); + let digest = review.lifetime.checkout_resources_digest(&[]).unwrap(); + assert_eq!(review.checkout_resources_digest.as_ref(), Some(&digest)); + let confirmation = super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "resource-bound-cancel".into(), + }; + *resources.lock().unwrap() = None; + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .is_err() + ); + assert!( + runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap() + .is_none() + ); + *resources.lock().unwrap() = Some(vec![]); + let receipt = runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .unwrap(); + assert_eq!(receipt.checkout_resources_digest, Some(digest)); + *resources.lock().unwrap() = None; + assert_eq!( + runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap(), + Some(receipt) + ); + assert!( + runtime + .snapshot_preparation_acknowledge(&session, &id, &confirmation.request_id) + .await + .is_err() + ); + runtime.shutdown().await; + base.shutdown().await; +} + #[tokio::test] async fn resource_publication_failure_retains_unknown_reservation_without_origin_or_source_work() { let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; @@ -1207,6 +1401,7 @@ async fn resource_publication_failure_retains_unknown_reservation_without_origin permits: permits.clone(), foreign: false, reject_resources: true, + resources: Default::default(), }), ); let handle = runtime.inner.handle(&session).await.unwrap(); @@ -1245,6 +1440,7 @@ async fn foreign_lifetime_proof_is_rejected_before_any_native_reservation() { permits: permits.clone(), foreign: true, reject_resources: false, + resources: Default::default(), }), ); let handle = runtime.inner.handle(&session).await.unwrap(); diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index 716944b46..de0d0bd01 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -23,30 +23,7 @@ fn validate_checkouts( binding: &SnapshotLifetimeBinding, resources: &[cyber_core::worktrees::CheckoutBinding], ) -> Result<(), String> { - let owner = binding.checkout_owner()?; - if resources.len() > 4096 - || binding - .owner - .worktree_id - .as_ref() - .is_some_and(|id| !resources.iter().any(|resource| &resource.worktree_id == id)) - || binding.owner.worktree_id.is_none() && !resources.is_empty() - { - return Err("Snapshot checkout resource set is incomplete or exceeds limits".into()); - } - let mut seen = std::collections::HashSet::new(); - for resource in resources { - if resource.version != 2 - || resource.owner_id != binding.owner.activity_id - || resource.lifetime.as_ref() != Some(&owner) - || !seen.insert((&resource.common_dir, &resource.worktree_id)) - { - return Err( - "Snapshot checkout resource set has foreign ownership or duplicates".into(), - ); - } - } - Ok(()) + binding.checkout_resources_digest(resources).map(drop) } pub(crate) struct Lifetime { @@ -150,6 +127,14 @@ impl SnapshotLifetime for Lifetime { return Err("Foreign preparation receipt".into()); } self.verify_preparation(receipt.origin.as_ref())?; + if let Some(expected) = &receipt.checkout_resources_digest { + let resources = self + .checkout_bindings()? + .ok_or("Missing consumed checkout resources")?; + if self.binding.checkout_resources_digest(&resources)? != *expected { + return Err("Preparation receipt checkout resources changed".into()); + } + } if receipt.origin.is_some() { let store = Store::recover_existing( &self.data, @@ -415,6 +400,21 @@ impl Store { } Ok(phase) } + pub(super) fn verify_consumed_resources( + &self, + receipt: &cyber_server::runtime::SnapshotPreparationReceipt, + ) -> Result<(), String> { + if let Some(expected) = &receipt.checkout_resources_digest { + let phase = self.read_source_phase(&receipt.lifetime)?; + let resources = phase + .checkout_bindings + .ok_or("Missing consumed checkout resources")?; + if receipt.lifetime.checkout_resources_digest(&resources)? != *expected { + return Err("Preparation receipt checkout resources changed".into()); + } + } + Ok(()) + } pub(super) fn verify_native_preparation( &self, binding: &SnapshotLifetimeBinding, diff --git a/crates/cyber-snapshot/src/copy/preparation.rs b/crates/cyber-snapshot/src/copy/preparation.rs index e12e9f775..b50b050b9 100644 --- a/crates/cyber-snapshot/src/copy/preparation.rs +++ b/crates/cyber-snapshot/src/copy/preparation.rs @@ -34,6 +34,7 @@ impl Store { receipt: &SnapshotPreparationReceipt, ) -> Result<(), String> { self.verify_native_preparation(&receipt.lifetime)?; + self.verify_consumed_resources(receipt)?; let origin = receipt .origin .as_ref() @@ -84,6 +85,7 @@ impl Store { serde_json::from_slice(&storage::read(&directory, FILE, LIMIT)?) .map_err(storage::fail)?; self.verify_native_preparation(&receipt.lifetime)?; + self.verify_consumed_resources(&receipt)?; let origin = receipt .origin .as_ref() @@ -160,6 +162,7 @@ mod tests { session_sequence: 8, origin: Some(origin), source_admitted: Some(false), + checkout_resources_digest: None, }; (temp, store, lifetime, receipt) } @@ -171,6 +174,10 @@ mod tests { std::os::unix::fs::symlink(&target, store.root.join("source")).unwrap(); assert!(store.ensure_recovered().is_err()); drop(store); + // A historical receipt still reconciles its original journal metadata; + // publishing resources does not promote it to checkout acknowledgement. + lifetime.bind_checkouts(&[]).unwrap(); + assert!(receipt.checkout_resources_digest.is_none()); lifetime.acknowledge_preparation(&receipt).unwrap(); lifetime.acknowledge_preparation(&receipt).unwrap(); let store = Store::recover_existing( @@ -198,6 +205,45 @@ mod tests { ); assert_eq!(std::fs::read(target).unwrap(), b"later owner bytes"); } + #[test] + fn resource_digest_must_match_native_publication_before_metadata_acknowledgement() { + let (temp, store, mut lifetime, mut receipt) = setup(); + let root = store.root.clone(); + let options = store.options.clone(); + let directory = store + .journals + .open_dir_nofollow("restore_preparation") + .unwrap(); + drop(store); + lifetime.bind_checkouts(&[]).unwrap(); + receipt.checkout_resources_digest = Some("0".repeat(64)); + assert!(lifetime.acknowledge_preparation(&receipt).is_err()); + assert!(directory.symlink_metadata(FILE).is_err()); + receipt.checkout_resources_digest = + Some(receipt.lifetime.checkout_resources_digest(&[]).unwrap()); + lifetime.acknowledge_preparation(&receipt).unwrap(); + let recorded: SnapshotPreparationReceipt = + serde_json::from_slice(&storage::read(&directory, FILE, LIMIT).unwrap()).unwrap(); + assert_eq!(recorded, receipt); + let store = + Store::recover_existing(&temp.path().join("data"), root.to_str().unwrap(), options) + .unwrap(); + assert_eq!( + store + .preparation_cancellation("restore_preparation") + .unwrap(), + Some(receipt) + ); + let mut corrupt = recorded; + corrupt.checkout_resources_digest = Some("0".repeat(64)); + storage::atomic(&directory, FILE, &serde_json::to_vec(&corrupt).unwrap()).unwrap(); + assert!( + store + .preparation_cancellation("restore_preparation") + .is_err() + ); + } + #[test] fn contradictory_intent_and_admitted_receipt_never_publish_cancellation_metadata() { let (_temp, store, _lifetime, mut receipt) = setup(); diff --git a/crates/cyber-store/migrations/20261107010000_snapshot_consumed_resources.sql b/crates/cyber-store/migrations/20261107010000_snapshot_consumed_resources.sql new file mode 100644 index 000000000..1f0f45e63 --- /dev/null +++ b/crates/cyber-store/migrations/20261107010000_snapshot_consumed_resources.sql @@ -0,0 +1,3 @@ +-- Preparation receipts bind the complete acquired native checkout set. +-- Previous writers cannot discard that consuming evidence. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index 01f50f701..986ea600d 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -193,6 +193,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261107000000_snapshot_checkout_resources", sql: include_str!("../migrations/20261107000000_snapshot_checkout_resources.sql"), }, + Migration { + id: "20261107010000_snapshot_consumed_resources", + sql: include_str!("../migrations/20261107010000_snapshot_consumed_resources.sql"), + }, ]; pub(crate) enum Outcome { @@ -480,10 +484,41 @@ mod tests { let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { panic!("previous writer accepted native resource publication"); }; - assert_eq!(unknown, ["20261107000000_snapshot_checkout_resources"]); + assert_eq!( + unknown, + MIGRATIONS[at..] + .iter() + .map(|migration| migration.id) + .collect::>() + ); assert_eq!(std::fs::read(phase).unwrap(), evidence); } + #[test] + fn consumed_resource_receipts_reject_previous_writers_and_preserve_event_bytes() { + let mut db = Connection::open_in_memory().unwrap(); + let at = MIGRATIONS + .iter() + .position(|migration| migration.id == "20261107010000_snapshot_consumed_resources") + .unwrap(); + migrate(&mut db, &MIGRATIONS[..at]).unwrap(); + let evidence = r#"{"checkout_resources_digest":"retained","activity_id":"op_original"}"#; + db.execute("INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('evt_resources','op_receipt',0,'snapshot.preparation.cancelled.1',?1,1)", [evidence]).unwrap(); + migrate(&mut db, MIGRATIONS).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { + panic!("previous writer accepted consumed checkout evidence"); + }; + assert_eq!(unknown, ["20261107010000_snapshot_consumed_resources"]); + let retained: String = db + .query_row( + "SELECT data FROM event WHERE id='evt_resources'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(retained, evidence); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index c32c7fad2..1b6cef02b 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1372,6 +1372,15 @@ impl ToolHost for BuiltinHost { Box::pin(self.claim_snapshot_worktree_location_bound(info, activity, lifetime, cancel)) } + fn reacquire_snapshot_location( + &self, + info: &cyber_server::runtime::SessionInfo, + lifetime: &cyber_server::runtime::SnapshotLifetimeBinding, + bindings: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result, String> { + self.reacquire_snapshot_worktree_location(info, lifetime, bindings) + } + fn prepare_child_continuation<'a>( &'a self, parent: &'a cyber_server::runtime::SessionInfo, diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index 04359b2a1..f26ff0f7d 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -283,6 +283,38 @@ impl BuiltinHost { .await } + pub(crate) fn reacquire_snapshot_worktree_location( + &self, + info: &SessionInfo, + lifetime: &cyber_server::runtime::SnapshotLifetimeBinding, + bindings: &[cyber_core::worktrees::CheckoutBinding], + ) -> Result, String> { + lifetime.checkout_resources_digest(bindings)?; + if lifetime.owner.session_id != info.id + || lifetime.owner.worktree_id != info.worktree_id + || std::fs::canonicalize(&info.directory).map_err(|error| error.to_string())? + != lifetime.owner.directory + { + return Err("Foreign snapshot checkout reacquisition context".into()); + } + let locations = Repository::managed_locations_at(Path::new(&info.directory)) + .map_err(|error| error.to_string())?; + if locations.len() != bindings.len() { + return Err("Snapshot checkout resource inventory changed".into()); + } + let owner = lifetime.checkout_owner()?; + let claims = locations + .iter() + .zip(bindings) + .map(|((repository, managed), binding)| { + repository + .reacquire_snapshot(managed, binding, &owner) + .map_err(|error| error.to_string()) + }) + .collect::, _>>()?; + Ok(Box::new(ReacquiredSnapshotProof(claims))) + } + pub(crate) async fn claim_hook_test_location( &self, event: &cyber_core::hooks::HookEvent, @@ -1280,6 +1312,19 @@ impl cyber_server::runtime::SnapshotLocationGuard for ActiveSnapshotLocation { } } +struct ReacquiredSnapshotProof(Vec); +impl cyber_server::runtime::SnapshotLocationGuard for ReacquiredSnapshotProof { + fn verify(&self) -> Result<(), String> { + self.0 + .iter() + .try_for_each(|claim| claim.verify().map_err(|error| error.to_string())) + } + fn bindings(&self) -> Result, String> { + self.verify()?; + Ok(self.0.iter().map(|claim| claim.binding().clone()).collect()) + } +} + struct SnapshotWorktreeProof(Vec); impl cyber_server::runtime::SnapshotLocationGuard for SnapshotWorktreeProof { fn verify(&self) -> Result<(), String> { diff --git a/crates/cyber-tools/tests/worktree_setup.rs b/crates/cyber-tools/tests/worktree_setup.rs index 07d7ac43b..1b2e6f74c 100644 --- a/crates/cyber-tools/tests/worktree_setup.rs +++ b/crates/cyber-tools/tests/worktree_setup.rs @@ -1992,7 +1992,60 @@ async fn lifetime_bound_checkout_host_preserves_context_and_refuses_foreign_requ serde_json::to_value(&bindings[0]).unwrap() ); assert_eq!(raw["settled"], true); - assert_eq!(std::fs::read_dir(directory).unwrap().count(), before + 1); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), before + 1); + let mut exited = lifetime.clone(); + exited.owner.activity_id = "op_exited".into(); + let lease = flow + .f + .host + .claim_snapshot_location_bound(&info, "op_exited", Some(&exited), CancellationToken::new()) + .await + .unwrap(); + let acquired = lease.snapshot_bindings().unwrap(); + assert!( + flow.f + .host + .reacquire_snapshot_location(&info, &exited, &acquired) + .is_err() + ); + drop(lease); + let retained = std::fs::read(directory.join("op_exited.lock")).unwrap(); + assert_eq!( + serde_json::from_slice::(&retained).unwrap()["settled"], + false + ); + let proof = flow + .f + .host + .reacquire_snapshot_location(&info, &exited, &acquired) + .unwrap(); + proof.verify().unwrap(); + assert_eq!(proof.bindings().unwrap(), acquired); + assert!( + flow.f + .host + .reacquire_snapshot_location(&info, &exited, &acquired) + .is_err() + ); + assert!( + flow.f + .host + .reacquire_snapshot_location(&info, &exited, &[]) + .is_err() + ); + let mut foreign = exited.clone(); + foreign.owner.session_created_id.push_str("_foreign"); + assert!( + flow.f + .host + .reacquire_snapshot_location(&info, &foreign, &acquired) + .is_err() + ); + drop(proof); + assert_eq!( + std::fs::read(directory.join("op_exited.lock")).unwrap(), + retained + ); flow.runtime.shutdown().await; } diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 830bee7f1..6ded8ebe2 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -187,3 +187,12 @@ Publication failure preserves the unknown reservation; no new native acknowledge or cleanup authority is granted. Interrupted-claim consuming receipts and native reconciliation remain the next recovery work, alongside the existing Git, reclamation and platform/client acceptance obligations. + + +Preparation receipts now consume a digest of the complete acquired checkout set, +with actual original claim locks retained through paired commit/reply. CLI/App +reacquire via an explicit read-only host port; lookup remains database-only and +journal metadata reconciliation checks the committed resource binding. Original +native checkout acknowledgement, partial-constructor recovery, Git journaling and +reference-aware cleanup remain required before the complete recovery increment is +reviewable for delivery. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index f723c46cf..ced8d4108 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1463,3 +1463,86 @@ the disposal fix, including both pre-origin/pre-source service kills and partial keep/rollback restart cases. Final formatting/diff checks passed. Review found no remaining critical/major issue within this increment; the broader recovery and platform/client acceptance gaps above remain required before delivery. + + +## Consuming original acquired checkout evidence (2026-11-07) + +Preparation workers now load the complete native source-phase resource set under +the original exited worker lifetime lock. The explicit production host port +reacquires the exact existing claims as read-only `SnapshotClaim` values, matching +current managed creation/context and the complete ordered persisted resource set. +It performs no Git/process launch, claim creation, generic settlement or source +mutation. The returned native proof retains original lock descriptors and metadata +pins. CLI and App implement the same host port; a host without managed support +refuses rather than replacing original evidence with a fresh claim. + +A versioned SHA256 digest binds the immutable lifetime and complete acquired set. +Validation checks version, owner/context, nearest checkout presence, empty unmanaged +sets, inventory bound and duplicates. The native publication validator shares this +encoding/validation. Fixed known-answer coverage pins both lifetime and resource +encodings, including the empty complete set; changed lifetime identities change the +digest and legacy lifetimes cannot authorize it. The review fingerprint includes +the resource digest, and the paired original/request cancellation receipt stores +it. Runtime rechecks the published set and actual retained proof inside the Store +writer, returning the worker through COMMIT and observed reply. Original native +locks remain held until that worker unwinds. + +Explicit journal metadata acknowledgement verifies the committed resource digest +against native publication. Native tombstone creation and loading both validate +that binding, so internal Store entry points cannot bypass it. Historical receipts +without a resource digest retain their existing journal-only authority and gain no +checkout acknowledgement capability. Lost-reply/request lookup remains database-only +and never reacquires native resources or replays source work. A compatibility catalog +marker prevents older writers from discarding consumed-resource evidence; OpenAPI +and generated TypeScript types include the optional review/receipt digest. + +Regression coverage uses the actual managed production host to retain and reacquire +an unknown original claim, refuse active/foreign/incomplete evidence and preserve +claim bytes. Runtime instrumentation changes native publication after review, +verifies failed confirmation and absent receipt, restores the exact set for positive +confirmation, then changes evidence again while database lookup still succeeds and +explicit metadata reconciliation refuses. Separate changed-proof and Stop arrivals at the consuming +writer's retained-checkout verification roll back both owners; the final Stop check +follows all native proof verification. The same unchanged review succeeds after +valid proof returns under a fresh preparation owner. Native tests refuse a mismatched digest +before publishing journal metadata and preserve old journal reconciliation without +promoting its receipt. Real CLI kill/restart tests compare review/receipt resource +binding and require the exact empty-set digest after published origins. + +The code-review skill is applied to all changed runtime/native/Store/adapter, +API/SDK, test and specification files. Review checks native-only writer callbacks, +original evidence rather than fresh claims, lifetime/resource ownership, bounded +encoding, immutable digest framing, held descriptors through commit/reply, both +native tombstone entry points, legacy journal compatibility and absence of native +claim acknowledgement or cleanup authority. Native unknown-claim settlement, +partial-construction resource completion, Git recovery, reclamation and complete +platform/client acceptance remain open with M2–M8 and parent tasks 2.4/3.2–3.4. + + +Verification on local macOS arm64: final server unit tests passed 426 cases with +one ignored fixture, including changed original proof and Stop during consuming +verification. The 43-case focused snapshot run also passed. All 85 snapshot tests +passed with one ignored subprocess fixture; after adding the native reader +corruption assertion, its focused digest/tombstone regression passed again. All +63 Store tests passed, including prior-catalog refusal and exact retained event +bytes. The selected production-host managed lifetime/reacquisition integration +passed (39 unrelated cases filtered). OpenAPI regeneration/current-document check +passed; SDK typechecking and 134 SDK tests passed, with 10 real-process SDK tests +skipped because CYBER_BIN was not set. Generated SDK consistency passed. Strict +pinned OpenSpec passed all 117 items and spec lint reports 0 errors/25 warnings. +Final all-target Clippy for server/Store/snapshot/tools/App/CLI passed with warnings +denied. An initial Clippy run found 18/15 complexity in the expanded CLI fixture; +extracting its receipt assertions into a helper resolved it without an allowance. +The initial compile's missing review field was fixed before successful validation. +Formatting and diff checks passed. Native sandbox deprecation and Darwin linker +warnings were observed without assigning their origin to main. Full-workspace +tests, native Linux/Windows execution, and live TUI/App recovery acceptance were +not run for this increment. The code/security review found no remaining critical +or major issue within this consuming-proof scope; native checkout acknowledgement +and the broader recovery acceptance obligations remain incomplete. + +Final real CLI verification passed all 16 branch/recovery cases after the consuming +Stop fix and assertion-helper extraction, including pre-origin and published-origin +service kills, restart/lookup, and partial keep/rollback. Their successful receipt +assertions require review/commit resource agreement and the exact complete empty-set +digest after published origins. No additional parent acceptance is claimed. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 0535ecaa9..33e667a80 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -388,3 +388,17 @@ This resource set is retained alongside source admission/origin and remains prot for later consuming receipt reconciliation. The current step does not acknowledge unknown claims or infer completeness from missing files; pre-publication construction cuts and complete reference/cleanup acceptance remain required. + + +### Consuming acquired-resource evidence + +Preparation review and its committed cancellation receipt bind a versioned digest +of the complete persisted checkout set, including the exact immutable lifetime. +Recovery reacquires those original claims read-only and retains their actual locks +and native metadata pins through the consuming Store commit and observed reply. +The writer revalidates both the retained set and native publication. Missing, +foreign, live or changed evidence refuses without a consuming receipt. Historical +receipts lacking the digest remain readable but cannot acquire new checkout +acknowledgement authority. Request lookup remains database-only. Explicit metadata +reconciliation must match the committed digest before accessing original claims. +This binding does not by itself acknowledge an unknown native checkout claim. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 3659d5e6f..a6f030890 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -116,6 +116,12 @@ - **AND** a changed set or post-admission addition SHALL refuse, source admission SHALL retain the published set, and incomplete evidence SHALL NOT gain reference-completeness authority - **AND** disposal before origin publication SHALL retain a lifetime-bound reservation as unknown until explicit exact-owner reconciliation +#### Scenario: Preparation receipt consumes exact acquired resources +- **WHEN** recovery cancels an exited preparation with a complete published checkout set +- **THEN** review and its durable receipt SHALL bind the exact lifetime and resource-set digest +- **AND** original claim locks and pins SHALL remain held through consuming commit and observed reply, with changed or live evidence refusing +- **AND** historical receipts without this binding SHALL gain no new checkout acknowledgement authority, while request lookup SHALL remain database-only + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index c2d3999f9..eccad53e9 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -222,3 +222,15 @@ retains references and refuses missing managed sets. Explicit empty unmanaged se remain distinct from absent constructor evidence. Unknown-claim acknowledgement, partial acquisition/publication cuts, Git journals, reclamation and full native/client acceptance remain open; no parent task is marked complete. + + +Consuming acquired-resource proof continuation: preparation review and cancellation +receipts now bind a versioned digest of the exact lifetime and complete acquired +checkout set. The production CLI/App host port reacquires original claims read-only; +recovery retains and revalidates those locks/pins and native reference publication +through the paired Store commit and observed reply. Missing/changed/live/foreign +proof refuses without a consuming receipt. Database-only lookup remains independent +of native evidence; explicit journal metadata acknowledgement verifies the committed +digest. Historical digest-less receipts retain only their existing journal authority. +Native unknown-claim acknowledgement is still required: this continuation does not +settle a checkout claim, delete construction evidence or accept any parent task. diff --git a/sdk/openapi.json b/sdk/openapi.json index c1f9485f1..cec042254 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7605,6 +7605,12 @@ "type": "null" } ] + }, + "checkout_resources_digest": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -7752,6 +7758,12 @@ "boolean", "null" ] + }, + "checkout_resources_digest": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 5de2c677f..ce0e2daaf 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -4766,6 +4766,7 @@ export type SnapshotPreparationReceipt = { session_sequence: number; origin?: SnapshotOrigin | null; source_admitted?: boolean | null; + checkout_resources_digest?: string | null; }; export type SnapshotPreparationReview = { @@ -4776,6 +4777,7 @@ export type SnapshotPreparationReview = { confirmation: string; warning: string; origin?: SnapshotOrigin | null; + checkout_resources_digest?: string | null; }; /** Retained filesystem evidence; completion does not acknowledge a Session/native receipt. */ From 1564463d49205e70634fbe326b70f4b0c6a311a7 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 01:36:14 -0400 Subject: [PATCH 18/87] Acknowledge exact pre-effect snapshot checkout claims after durable settlement --- crates/cyber-core/src/worktrees.rs | 3 +- crates/cyber-core/src/worktrees/activity.rs | 26 +- .../src/worktrees/activity/snapshot.rs | 44 +- .../activity/snapshot/acknowledgement.rs | 376 ++++++++++++++++++ .../snapshot/acknowledgement/tests.rs | 268 +++++++++++++ .../activity/snapshot/publication.rs | 6 +- crates/cyber-core/src/worktrees/removal.rs | 51 ++- .../tests/worktree_repository/activity.rs | 100 +++++ crates/cyber-server/src/runtime/host.rs | 27 ++ .../src/runtime/snapshot_lifetime.rs | 28 ++ .../src/runtime/snapshot_preparation.rs | 48 ++- .../cyber-server/src/runtime/snapshot_work.rs | 30 +- .../src/runtime/snapshot_work_tests.rs | 198 ++++++++- crates/cyber-snapshot/src/copy/lifetime.rs | 26 ++ crates/cyber-snapshot/src/copy/preparation.rs | 13 + ...000_snapshot_checkout_acknowledgements.sql | 3 + crates/cyber-store/src/migrate.rs | 42 +- crates/cyber-tools/src/worktrees.rs | 42 ++ crates/cyber-tools/tests/worktree_setup.rs | 36 +- docs/goal-delivery.md | 11 + docs/rewind.md | 34 +- .../implementation.md | 103 +++++ .../managed-recovery.md | 50 ++- .../specs/snapshots-checkpoints/spec.md | 8 + .../complete-daily-coding-surfaces/tasks.md | 15 + 25 files changed, 1543 insertions(+), 45 deletions(-) create mode 100644 crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs create mode 100644 crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs create mode 100644 crates/cyber-store/migrations/20261107020000_snapshot_checkout_acknowledgements.sql diff --git a/crates/cyber-core/src/worktrees.rs b/crates/cyber-core/src/worktrees.rs index 6c6adc583..de857ba43 100644 --- a/crates/cyber-core/src/worktrees.rs +++ b/crates/cyber-core/src/worktrees.rs @@ -9,7 +9,8 @@ mod removal; pub(crate) mod repository; mod setup; pub use activity::{ - CheckoutActivity, CheckoutBinding, CheckoutLease, CheckoutOwner, SnapshotClaim, + CheckoutAcknowledgement, CheckoutActivity, CheckoutBinding, CheckoutLease, CheckoutOwner, + SnapshotClaim, }; pub use changes::{ChangedFile, Changes}; pub use includes::IncludedFile; diff --git a/crates/cyber-core/src/worktrees/activity.rs b/crates/cyber-core/src/worktrees/activity.rs index 9b4b960ec..1ba168236 100644 --- a/crates/cyber-core/src/worktrees/activity.rs +++ b/crates/cyber-core/src/worktrees/activity.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use super::{GitExecution, Managed, Name, RemovalActivity, Repository, RepositoryLock}; mod snapshot; -pub use snapshot::{CheckoutBinding, CheckoutOwner, SnapshotClaim}; +pub use snapshot::{CheckoutAcknowledgement, CheckoutBinding, CheckoutOwner, SnapshotClaim}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct UseRecord { @@ -99,19 +99,26 @@ pub struct CheckoutActivity; struct RemovalFence { _files: Vec, + _snapshots: Vec>, } impl RemovalActivity for CheckoutActivity { fn reserve(&self, managed: &Managed) -> io::Result> { let directory = use_directory(managed, false)?; if !directory.exists() { - return Ok(Box::new(RemovalFence { _files: Vec::new() })); + return Ok(Box::new(RemovalFence { + _files: Vec::new(), + _snapshots: vec![], + })); } let mut paths = std::fs::read_dir(&directory)? .map(|entry| entry.map(|entry| entry.path())) .collect::>>()?; paths.sort(); + let (paths, inventory) = snapshot::acknowledgement::Inventory::split(paths)?; let mut files = Vec::with_capacity(paths.len()); + let mut snapshots = vec![]; + let mut acknowledged = std::collections::HashMap::new(); for path in paths { let session = path .file_stem() @@ -128,12 +135,21 @@ impl RemovalActivity for CheckoutActivity { let mut file = LockedRecord(file); let record = read_record(&mut file)?; check_record(&record, managed, session)?; - if !record.settled { + let proof = snapshot::acknowledgement::for_removal(managed, &record, &file)?; + if !record.settled && proof.is_none() { return Err(unknown(session)); } + if let Some((binding, proof)) = proof { + acknowledged.insert(session.to_owned(), binding); + snapshots.push(proof); + } files.push(file); } - Ok(Box::new(RemovalFence { _files: files })) + files.extend(inventory.retain(managed, &acknowledged)?); + Ok(Box::new(RemovalFence { + _files: files, + _snapshots: snapshots, + })) } } @@ -288,7 +304,7 @@ impl Repository { } } -fn safe_id(id: &str, prefix: &str) -> io::Result<()> { +pub(super) fn safe_id(id: &str, prefix: &str) -> io::Result<()> { if !crate::ids::has_prefix(id, prefix) || id.len() > 128 || !id diff --git a/crates/cyber-core/src/worktrees/activity/snapshot.rs b/crates/cyber-core/src/worktrees/activity/snapshot.rs index 2b23129b8..c5f9973c4 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot.rs @@ -1,6 +1,8 @@ //! Exact checkout metadata pins. A binding alone never grants acknowledgement. use super::{Managed, UseRecord, invalid}; +pub(super) mod acknowledgement; mod publication; +pub use acknowledgement::CheckoutAcknowledgement; use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, OpenOptions}; pub(super) use publication::publish; @@ -47,6 +49,7 @@ pub struct CheckoutOwner { /// No generic settlement, creation or source mutation capability is exposed. pub struct SnapshotClaim { lease: super::CheckoutLease, + acknowledgement: Option, } impl SnapshotClaim { pub fn binding(&self) -> &CheckoutBinding { @@ -57,7 +60,14 @@ impl SnapshotClaim { .expect("reacquired binding") } pub fn verify(&self) -> io::Result<()> { - self.lease.verify_snapshot() + self.lease.verify_snapshot()?; + if let Some(acknowledged) = &self.acknowledgement { + acknowledged.verify( + self.lease.snapshot_pins.as_ref().expect("retained pins"), + self.binding(), + )?; + } + Ok(()) } } @@ -103,7 +113,10 @@ pub(super) fn reacquire( snapshot_pins: Some(pins), }; lease.verify_snapshot()?; - Ok(SnapshotClaim { lease }) + Ok(SnapshotClaim { + lease, + acknowledgement: None, + }) } impl CheckoutOwner { fn validate_fields(&self) -> io::Result<()> { @@ -140,6 +153,7 @@ pub(super) struct Pins { directory: Dir, creations: Dir, creation: File, + archived: bool, } fn open_file(directory: &Dir, name: &str) -> io::Result { @@ -239,6 +253,17 @@ fn digest(bytes: &[u8]) -> String { impl Pins { fn open(common_dir: &Path, worktree: &str, name: &str) -> io::Result { + Self::open_creation(common_dir, worktree, name, false) + } + fn open_for_removal(common_dir: &Path, worktree: &str, name: &str) -> io::Result { + Self::open_creation(common_dir, worktree, name, true) + } + fn open_creation( + common_dir: &Path, + worktree: &str, + name: &str, + allow_archived: bool, + ) -> io::Result { if std::fs::symlink_metadata(common_dir)? .file_type() .is_symlink() @@ -249,13 +274,20 @@ impl Pins { let root = common.open_dir_nofollow("cyber-worktree-activity")?; let directory = root.open_dir_nofollow(worktree)?; let creations = common.open_dir_nofollow("cyber-worktrees")?; - let creation = open_file(&creations, &format!("{name}.json"))?; + let (creation, archived) = match open_file(&creations, &format!("{name}.json")) { + Ok(file) => (file, false), + Err(error) if allow_archived && error.kind() == io::ErrorKind::NotFound => { + (open_file(&creations, &format!("{worktree}.removed"))?, true) + } + Err(error) => return Err(error), + }; Ok(Self { common, root, directory, creations, creation, + archived, }) } pub(super) fn capture( @@ -342,7 +374,11 @@ impl Pins { return Err(invalid("Snapshot checkout descriptor changed")); } self.matches(binding)?; - let current = Self::open(&binding.common_dir, &binding.worktree_id, &binding.name)?; + let current = if self.archived { + Self::open_for_removal(&binding.common_dir, &binding.worktree_id, &binding.name)? + } else { + Self::open(&binding.common_dir, &binding.worktree_id, &binding.name)? + }; current.matches(binding)?; if digest(&bytes(¤t.creation)?) != binding.creation_digest { return Err(invalid("Snapshot checkout creation contents changed")); diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs new file mode 100644 index 000000000..3490cd594 --- /dev/null +++ b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs @@ -0,0 +1,376 @@ +//! Durable disposition of an exact claim, separate from its immutable evidence. +use super::super::{LockedRecord, Managed, UseRecord, invalid}; +use super::{CheckoutBinding, Pins, SnapshotClaim, contents, identity, open_file, regular}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::OpenOptions; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, HashSet}, + fs::File, + io::{self, Write}, + path::PathBuf, +}; + +const LIMIT: usize = 32 * 1024; +const SUFFIX: &str = ".snapshot-ack"; + +/// Trusted adapter authority issued only after observing the exact durable receipt. +/// Digests alone never authorize acquisition of a different or historical claim. +pub struct CheckoutAcknowledgement { + binding: CheckoutBinding, + resources_digest: String, + receipt_digest: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Record { + version: u32, + binding: CheckoutBinding, + resources_digest: String, + receipt_digest: String, + file_identity: String, + staging_name: String, +} + +pub(super) struct Acknowledged { + file: LockedRecord, + record: Record, +} + +fn hash(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} +fn stage_prefix(owner: &str) -> String { + format!("ack_preparing_{}_{owner}_", owner.len()) +} +fn stage_owner(name: &str) -> io::Result<&str> { + let tail = name + .strip_prefix("ack_preparing_") + .ok_or_else(|| invalid("Invalid acknowledgement stage"))?; + let (length, tail) = tail + .split_once('_') + .ok_or_else(|| invalid("Invalid acknowledgement stage length"))?; + let length: usize = length.parse().map_err(invalid)?; + let owner = tail + .get(..length) + .ok_or_else(|| invalid("Invalid acknowledgement stage owner"))?; + super::super::safe_id(owner, "op")?; + let nonce = tail + .get(length..) + .and_then(|tail| tail.strip_prefix('_')) + .ok_or_else(|| invalid("Invalid acknowledgement stage nonce"))?; + if nonce.len() != 26 + || !nonce.bytes().all(|byte| byte.is_ascii_alphanumeric()) + || name != format!("{}{nonce}", stage_prefix(owner)) + { + return Err(invalid("Invalid acknowledgement construction name")); + } + Ok(owner) +} +impl Acknowledged { + fn open(pins: &Pins, binding: &CheckoutBinding) -> io::Result { + let file = open_file(&pins.directory, &format!("{}{SUFFIX}", binding.owner_id))?; + if !super::super::try_lock(&file)? { + return Err(invalid("Checkout acknowledgement is still active")); + } + let file = LockedRecord(file); + let record: Record = serde_json::from_slice(&contents(&file, LIMIT)?).map_err(invalid)?; + let acknowledged = Self { file, record }; + acknowledged.verify(pins, binding)?; + Ok(acknowledged) + } + pub(super) fn verify(&self, pins: &Pins, binding: &CheckoutBinding) -> io::Result<()> { + if self.record.version != 1 + || self.record.binding != *binding + || binding.version != 2 + || binding.lifetime.is_none() + || !hash(&self.record.resources_digest) + || !hash(&self.record.receipt_digest) + || stage_owner(&self.record.staging_name)? != binding.owner_id + { + return Err(invalid("Checkout acknowledgement ownership changed")); + } + regular(&self.file)?; + let current = if pins.archived { + Pins::open_for_removal(&binding.common_dir, &binding.worktree_id, &binding.name)? + } else { + Pins::open(&binding.common_dir, &binding.worktree_id, &binding.name)? + }; + pins.matches(binding)?; + current.matches(binding)?; + let routed = open_file(¤t.directory, &format!("{}{SUFFIX}", binding.owner_id))?; + let observed: Record = + serde_json::from_slice(&contents(&self.file, LIMIT)?).map_err(invalid)?; + if identity(&self.file)? != self.record.file_identity + || identity(&routed)? != self.record.file_identity + || observed != self.record + { + return Err(invalid( + "Checkout acknowledgement route or contents changed", + )); + } + Ok(()) + } +} + +impl CheckoutAcknowledgement { + pub fn new( + binding: &CheckoutBinding, + resources_digest: &str, + receipt_digest: &str, + ) -> io::Result { + if binding.version != 2 + || binding.lifetime.is_none() + || !hash(resources_digest) + || !hash(receipt_digest) + { + return Err(invalid( + "Checkout acknowledgement lacks exact consuming receipt proof", + )); + } + Ok(Self { + binding: binding.clone(), + resources_digest: resources_digest.into(), + receipt_digest: receipt_digest.into(), + }) + } + /// Native metadata only. The caller must retain matching exited lifetime and + /// complete-resource proof through the observed Store commit and this call. + pub fn publish(&self, claim: &mut SnapshotClaim) -> io::Result<()> { + self.publish_before(claim, || Ok(())) + } + fn publish_before( + &self, + claim: &mut SnapshotClaim, + before: impl FnOnce() -> io::Result<()>, + ) -> io::Result<()> { + if !cfg!(any(target_os = "macos", target_os = "linux")) { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "Durable snapshot checkout acknowledgement is unavailable", + )); + } + self.retain_existing(claim)?; + if claim.acknowledgement.is_none() { + let acknowledged = self.create(claim, before)?; + claim.acknowledgement = Some(acknowledged); + } + let acknowledged = claim + .acknowledgement + .as_ref() + .expect("acknowledgement installed"); + let pins = claim + .lease + .snapshot_pins + .as_ref() + .expect("retained checkout pins"); + super::publication::sync_file(&acknowledged.file)?; + super::publication::sync_directory(&pins.directory)?; + claim.verify() + } + /// Read-only candidate verification. Retains an existing exact marker through + /// the consuming transaction; never creates acknowledgement metadata. + pub fn retain_existing(&self, claim: &mut SnapshotClaim) -> io::Result<()> { + claim.verify()?; + if claim.binding() != &self.binding { + return Err(invalid( + "Checkout acknowledgement differs from retained original claim", + )); + } + if let Some(acknowledged) = &claim.acknowledgement { + return self.matches(&acknowledged.record); + } + let pins = claim + .lease + .snapshot_pins + .as_ref() + .ok_or_else(|| invalid("Missing original checkout pins"))?; + match pins + .directory + .symlink_metadata(format!("{}{SUFFIX}", self.binding.owner_id)) + { + Ok(_) => { + let acknowledged = Acknowledged::open(pins, &self.binding)?; + self.matches(&acknowledged.record)?; + claim.acknowledgement = Some(acknowledged); + claim.verify() + } + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } + } + fn matches(&self, record: &Record) -> io::Result<()> { + if record.binding != self.binding + || record.resources_digest != self.resources_digest + || record.receipt_digest != self.receipt_digest + { + return Err(invalid( + "Checkout already has a different consuming receipt", + )); + } + Ok(()) + } + fn create( + &self, + claim: &SnapshotClaim, + before: impl FnOnce() -> io::Result<()>, + ) -> io::Result { + let pins = claim + .lease + .snapshot_pins + .as_ref() + .ok_or_else(|| invalid("Missing checkout pins"))?; + for (index, entry) in pins.directory.entries()?.enumerate() { + entry?; + if index >= 4095 { + return Err(invalid("Checkout acknowledgement inventory exceeds limit")); + } + } + let stage = format!( + "{}{}", + stage_prefix(&self.binding.owner_id), + ulid::Ulid::new() + ); + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .create_new(true) + .follow(FollowSymlinks::No); + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = pins.directory.open_with(&stage, &options)?.into_std(); + if !super::super::try_lock(&file)? { + return Err(invalid("Checkout acknowledgement construction is busy")); + } + let mut file = LockedRecord(file); + let record = Record { + version: 1, + binding: self.binding.clone(), + resources_digest: self.resources_digest.clone(), + receipt_digest: self.receipt_digest.clone(), + file_identity: identity(&file)?, + staging_name: stage.clone(), + }; + let bytes = serde_json::to_vec(&record).map_err(invalid)?; + if bytes.len() > LIMIT { + return Err(invalid("Checkout acknowledgement exceeds limit")); + } + file.write_all(&bytes)?; + super::publication::sync_file(&file)?; + super::publication::sync_directory(&pins.directory)?; + before()?; + claim.verify()?; + let routed = open_file(&pins.directory, &stage)?; + if identity(&routed)? != record.file_identity || contents(&file, LIMIT)? != bytes { + return Err(invalid("Checkout acknowledgement construction changed")); + } + super::publication::publish_name( + &pins.directory, + &stage, + &format!("{}{SUFFIX}", self.binding.owner_id), + )?; + super::publication::sync_directory(&pins.directory)?; + let acknowledged = Acknowledged { file, record }; + acknowledged.verify(pins, &self.binding)?; + Ok(acknowledged) + } +} + +pub(in crate::worktrees::activity) fn for_removal( + managed: &Managed, + record: &UseRecord, + file: &File, +) -> io::Result)>> { + let Some(binding) = &record.snapshot_binding else { + return Ok(None); + }; + let path = + super::super::use_directory(managed, false)?.join(format!("{}{SUFFIX}", record.owner_id)); + match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + Ok(_) => {} + } + let pins = Pins::open_for_removal(&managed.common_dir, &managed.id, &managed.name)?; + let observed: Managed = + serde_json::from_slice(&super::bytes(&pins.creation)?).map_err(invalid)?; + if observed != *managed || !observed.ready { + return Err(invalid("Acknowledged removal creation context differs")); + } + pins.verify(file, record)?; + let acknowledged = Acknowledged::open(&pins, binding)?; + super::publication::sync_file(&acknowledged.file)?; + super::publication::sync_directory(&pins.directory)?; + Ok(Some((binding.clone(), Box::new((acknowledged, pins))))) +} + +pub(in crate::worktrees::activity) struct Inventory { + markers: HashSet, + stages: Vec<(String, String)>, +} +impl Inventory { + pub(in crate::worktrees::activity) fn split( + paths: Vec, + ) -> io::Result<(Vec, Self)> { + if paths.len() > 4096 { + return Err(invalid("Checkout activity inventory exceeds limit")); + } + let mut inventory = Self { + markers: HashSet::new(), + stages: vec![], + }; + let mut claims = vec![]; + for path in paths { + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| invalid("Invalid checkout activity name"))?; + if let Some(owner) = name.strip_suffix(SUFFIX) { + super::super::safe_id(owner, "op")?; + inventory.markers.insert(owner.into()); + } else if name.starts_with("ack_preparing_") { + inventory + .stages + .push((stage_owner(name)?.into(), name.into())); + } else { + claims.push(path); + } + } + Ok((claims, inventory)) + } + pub(in crate::worktrees::activity) fn retain( + self, + managed: &Managed, + acknowledged: &HashMap, + ) -> io::Result> { + if self + .markers + .iter() + .any(|owner| !acknowledged.contains_key(owner)) + { + return Err(invalid("Orphan or unverified checkout acknowledgement")); + } + let mut files = vec![]; + for (owner, stage) in self.stages { + let binding = acknowledged + .get(&owner) + .ok_or_else(|| invalid("Unacknowledged checkout construction remains"))?; + let pins = Pins::open_for_removal(&managed.common_dir, &managed.id, &managed.name)?; + pins.matches(binding)?; + let file = open_file(&pins.directory, &stage)?; + if !super::super::try_lock(&file)? { + return Err(invalid("Checkout acknowledgement construction is active")); + } + files.push(LockedRecord(file)); + } + Ok(files) + } +} + +#[cfg(all(test, any(target_os = "macos", target_os = "linux")))] +mod tests; diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs new file mode 100644 index 000000000..89a730b2e --- /dev/null +++ b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs @@ -0,0 +1,268 @@ +use super::*; +use crate::worktrees::{CheckoutActivity, CheckoutOwner, RemovalActivity}; +use std::io::Read; + +fn fixture(root: &std::path::Path) -> Managed { + let common = root.join("common"); + let path = root.join("work"); + std::fs::create_dir_all(common.join("cyber-worktrees")).unwrap(); + std::fs::create_dir_all(&path).unwrap(); + let managed = Managed { + id: "wt_fixture".into(), + name: "fixture".into(), + path, + branch: "cyber/fixture".into(), + base: "main".into(), + common_dir: common, + ready: true, + included: vec![], + }; + std::fs::write( + root.join("common/cyber-worktrees/fixture.json"), + serde_json::to_vec(&managed).unwrap(), + ) + .unwrap(); + managed +} +fn directory(managed: &Managed) -> PathBuf { + managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id) +} +fn original(managed: &Managed) -> CheckoutBinding { + let owner = CheckoutOwner { + activity_id: "op_fixture".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: managed.path.clone(), + lifetime_digest: "a".repeat(64), + }; + let lease = + super::super::publication::publish(managed, &owner.activity_id, Some(owner.clone())) + .unwrap(); + lease.snapshot_binding().unwrap().clone() +} +fn claim(managed: &Managed, binding: &CheckoutBinding) -> SnapshotClaim { + super::super::reacquire(managed, binding, binding.lifetime.as_ref().unwrap()).unwrap() +} +fn authority(binding: &CheckoutBinding) -> CheckoutAcknowledgement { + CheckoutAcknowledgement::new(binding, &"a".repeat(64), &"b".repeat(64)).unwrap() +} + +#[test] +fn acknowledgement_is_exact_idempotent_and_preserves_original_and_independent_unknown_bytes() { + let temp = tempfile::tempdir().unwrap(); + let managed = fixture(&temp.path().canonicalize().unwrap()); + let binding = original(&managed); + let path = directory(&managed).join("op_fixture.lock"); + let original = std::fs::read(&path).unwrap(); + let mut proof = claim(&managed, &binding); + let authority = authority(&binding); + let mut legacy = binding.clone(); + legacy.version = 1; + legacy.lifetime = None; + assert!(CheckoutAcknowledgement::new(&legacy, &"a".repeat(64), &"b".repeat(64)).is_err()); + authority.retain_existing(&mut proof).unwrap(); + let marker = directory(&managed).join("op_fixture.snapshot-ack"); + assert!(!marker.exists()); + assert!(CheckoutActivity.reserve(&managed).is_err()); + authority.publish(&mut proof).unwrap(); + let bytes = std::fs::read(&marker).unwrap(); + authority.publish(&mut proof).unwrap(); + assert_eq!(std::fs::read(&marker).unwrap(), bytes); + assert_eq!(std::fs::read(&path).unwrap(), original); + assert!( + CheckoutAcknowledgement::new(&binding, &"a".repeat(64), &"c".repeat(64)) + .unwrap() + .retain_existing(&mut proof) + .is_err() + ); + assert!(CheckoutActivity.reserve(&managed).is_err()); // actual retained original lock + drop(proof); + let mut reloaded = claim(&managed, &binding); + authority.publish(&mut reloaded).unwrap(); + drop(reloaded); + drop(CheckoutActivity.reserve(&managed).unwrap()); + let mut foreign = managed.clone(); + foreign.path = temp.path().join("foreign-location"); + assert!(CheckoutActivity.reserve(&foreign).is_err()); + let independent = directory(&managed).join("ses_other.lock"); + let record = UseRecord { + worktree_id: managed.id.clone(), + owner_id: "ses_other".into(), + settled: false, + snapshot_binding: None, + }; + let bytes = serde_json::to_vec(&record).unwrap(); + std::fs::write(&independent, &bytes).unwrap(); + assert!(CheckoutActivity.reserve(&managed).is_err()); + assert_eq!(std::fs::read(independent).unwrap(), bytes); + assert_eq!(std::fs::read(path).unwrap(), original); +} + +#[test] +fn marker_replacement_alias_and_foreign_construction_never_gain_removal_authority() { + for case in [ + "replacement", + "alias", + "foreign-stage", + "unacknowledged-stage", + ] { + let temp = tempfile::tempdir().unwrap(); + let managed = fixture(&temp.path().canonicalize().unwrap()); + let binding = original(&managed); + let mut proof = claim(&managed, &binding); + let authority = authority(&binding); + if case != "unacknowledged-stage" { + authority.publish(&mut proof).unwrap(); + } + drop(proof); + let marker = directory(&managed).join("op_fixture.snapshot-ack"); + match case { + "replacement" => { + let bytes = std::fs::read(&marker).unwrap(); + std::fs::rename(&marker, marker.with_extension("retained")).unwrap(); + std::fs::write(&marker, &bytes).unwrap(); + assert!( + authority + .retain_existing(&mut claim(&managed, &binding)) + .is_err() + ); + assert_eq!(std::fs::read(marker).unwrap(), bytes); + } + "alias" => { + std::fs::hard_link(&marker, marker.with_extension("alias")).unwrap(); + assert!( + authority + .retain_existing(&mut claim(&managed, &binding)) + .is_err() + ); + } + _ => { + let owner = if case == "foreign-stage" { + "op_foreign" + } else { + "op_fixture" + }; + let stage = directory(&managed).join(format!( + "{}{}", + stage_prefix(owner), + ulid::Ulid::new() + )); + std::fs::write(&stage, b"incomplete retained metadata").unwrap(); + } + } + assert!(CheckoutActivity.reserve(&managed).is_err(), "{case}"); + } +} + +#[test] +fn namespace_replacement_before_acknowledgement_never_installs_a_marker_in_the_replacement() { + let temp = tempfile::tempdir().unwrap(); + let managed = fixture(&temp.path().canonicalize().unwrap()); + let binding = original(&managed); + let mut proof = claim(&managed, &binding); + let directory = directory(&managed); + let retained = directory.with_extension("retained"); + let result = authority(&binding).publish_before(&mut proof, || { + std::fs::rename(&directory, &retained)?; + std::fs::create_dir(&directory) + }); + assert!(result.is_err()); + assert!(!directory.join("op_fixture.snapshot-ack").exists()); + assert!(!retained.join("op_fixture.snapshot-ack").exists()); + assert_eq!(std::fs::read_dir(&retained).unwrap().count(), 2); + std::fs::remove_dir(&directory).unwrap(); // restore only the fixture's new empty replacement + std::fs::rename(&retained, &directory).unwrap(); + authority(&binding).publish(&mut proof).unwrap(); + drop(proof); + drop(CheckoutActivity.reserve(&managed).unwrap()); + assert_eq!(std::fs::read_dir(directory).unwrap().count(), 3); +} + +#[test] +#[ignore = "subprocess fixture invoked by acknowledgement kill test"] +fn acknowledgement_child() { + let root = PathBuf::from(std::env::var_os("CYBER_CHECKOUT_ACK_ROOT").unwrap()); + let managed: Managed = serde_json::from_slice( + &std::fs::read(root.join("common/cyber-worktrees/fixture.json")).unwrap(), + ) + .unwrap(); + let record: UseRecord = serde_json::from_slice( + &std::fs::read(directory(&managed).join("op_fixture.lock")).unwrap(), + ) + .unwrap(); + let binding = record.snapshot_binding.unwrap(); + let mut claim = claim(&managed, &binding); + authority(&binding) + .publish_before(&mut claim, || { + std::fs::write(root.join("ready"), b"prepared").unwrap(); + std::io::stdin().read_exact(&mut [0]).unwrap(); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn killed_acknowledgement_resumes_without_rewriting_claim_or_deleting_partial_stage() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let managed = fixture(&root); + let binding = original(&managed); + let path = directory(&managed).join("op_fixture.lock"); + let original = std::fs::read(&path).unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "worktrees::activity::snapshot::acknowledgement::tests::acknowledgement_child", + "--ignored", + ]) + .env("CYBER_CHECKOUT_ACK_ROOT", &root) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !root.join("ready").exists() { + assert!(child.try_wait().unwrap().is_none()); + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + super::super::reacquire(&managed, &binding, binding.lifetime.as_ref().unwrap()).is_err() + ); + let stage = std::fs::read_dir(directory(&managed)) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.file_name() + .unwrap() + .to_string_lossy() + .starts_with("ack_preparing_") + }) + .unwrap(); + let staged = File::open(&stage).unwrap(); + assert!(matches!( + staged.try_lock(), + Err(std::fs::TryLockError::WouldBlock) + )); + child.kill().unwrap(); + child.wait().unwrap(); + let bytes = std::fs::read(&stage).unwrap(); + assert!(CheckoutActivity.reserve(&managed).is_err()); + let mut proof = claim(&managed, &binding); + authority(&binding).publish(&mut proof).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), original); + assert_eq!(std::fs::read(&stage).unwrap(), bytes); + drop(proof); + let fence = CheckoutActivity.reserve(&managed).unwrap(); + assert!(matches!( + staged.try_lock(), + Err(std::fs::TryLockError::WouldBlock) + )); + drop(fence); + staged.try_lock().unwrap(); + staged.unlock().unwrap(); + assert_eq!(std::fs::read(&stage).unwrap(), bytes); +} diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/publication.rs b/crates/cyber-core/src/worktrees/activity/snapshot/publication.rs index 9ec18c3e3..779a04bc5 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot/publication.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot/publication.rs @@ -101,13 +101,13 @@ fn publish_before( Ok(lease) } -fn sync_file(file: &File) -> io::Result<()> { +pub(super) fn sync_file(file: &File) -> io::Result<()> { file.sync_all()?; #[cfg(target_os = "macos")] rustix::fs::fcntl_fullfsync(file)?; Ok(()) } -fn sync_directory(directory: &Dir) -> io::Result<()> { +pub(super) fn sync_directory(directory: &Dir) -> io::Result<()> { #[cfg(unix)] { let file = rustix::fs::openat( @@ -125,7 +125,7 @@ fn sync_directory(directory: &Dir) -> io::Result<()> { let _ = directory; Ok(()) } -fn publish_name(directory: &Dir, stage: &str, stable: &str) -> io::Result<()> { +pub(super) fn publish_name(directory: &Dir, stage: &str, stable: &str) -> io::Result<()> { #[cfg(any(target_os = "macos", target_os = "linux"))] { rustix::fs::renameat_with( diff --git a/crates/cyber-core/src/worktrees/removal.rs b/crates/cyber-core/src/worktrees/removal.rs index ee62ef6ea..d70d0b344 100644 --- a/crates/cyber-core/src/worktrees/removal.rs +++ b/crates/cyber-core/src/worktrees/removal.rs @@ -170,7 +170,7 @@ impl Repository { "Ownership changed during removal; recovery is required", )); } - std::fs::remove_file(&path)?; + retain_acknowledged_creation(self, &record.managed, &path)?; sync_parent(&path)?; } Err(error) if error.kind() == io::ErrorKind::NotFound => {} @@ -227,6 +227,55 @@ impl Repository { } } +fn retain_acknowledged_creation( + repo: &Repository, + managed: &Managed, + path: &Path, +) -> io::Result<()> { + super::activity::safe_id(&managed.id, "wt")?; + let activity = repo + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + let retained = match std::fs::read_dir(activity) { + Ok(entries) => { + let mut found = false; + for entry in entries { + found |= entry? + .path() + .extension() + .is_some_and(|extension| extension == "snapshot-ack"); + } + found + } + Err(error) if error.kind() == io::ErrorKind::NotFound => false, + Err(error) => return Err(error), + }; + if !retained { + return std::fs::remove_file(path); + } + let archived = path.with_file_name(format!("{}.removed", managed.id)); + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + rustix::fs::renameat_with( + rustix::fs::CWD, + path, + rustix::fs::CWD, + &archived, + rustix::fs::RenameFlags::NOREPLACE, + )?; + Ok(()) + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = archived; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Native acknowledged creation archival is unavailable", + )) + } +} + pub(super) fn check_admission(repo: &Repository, name: &Name) -> io::Result<()> { if read_removal(repo, name)?.is_some_and(|record| record.phase != RemovalPhase::Completed) { return Err(io::Error::other( diff --git a/crates/cyber-core/tests/worktree_repository/activity.rs b/crates/cyber-core/tests/worktree_repository/activity.rs index 15eb5f752..013a52634 100644 --- a/crates/cyber-core/tests/worktree_repository/activity.rs +++ b/crates/cyber-core/tests/worktree_repository/activity.rs @@ -1,6 +1,106 @@ use super::*; use cyber_core::worktrees::CheckoutActivity; +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[test] +fn exact_native_snapshot_acknowledgement_admits_removal_without_clearing_another_unknown_session() { + use cyber_core::worktrees::{CheckoutAcknowledgement, CheckoutOwner}; + for independent in [false, true] { + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("ack-snapshot").unwrap(), + &Settings::default(), + ) + .unwrap(); + let owner = CheckoutOwner { + activity_id: "op_bound".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: managed.path.clone(), + lifetime_digest: "a".repeat(64), + }; + if independent { + drop( + block_on(repository.claim(&fixture.execution, &managed, "ses_independent")) + .unwrap(), + ); + } + let lease = block_on(repository.claim_snapshot_bound(&fixture.execution, &managed, &owner)) + .unwrap(); + let binding = lease.snapshot_binding().unwrap().clone(); + drop(lease); + let path = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id) + .join("op_bound.lock"); + let original = std::fs::read(&path).unwrap(); + let creation = managed + .common_dir + .join("cyber-worktrees") + .join(format!("{}.json", managed.name)); + let creation_bytes = std::fs::read(&creation).unwrap(); + assert!( + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, true)) + .is_err() + ); + let mut proof = repository + .reacquire_snapshot(&managed, &binding, &owner) + .unwrap(); + CheckoutAcknowledgement::new(&binding, &"a".repeat(64), &"b".repeat(64)) + .unwrap() + .publish(&mut proof) + .unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), original); + assert!( + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, true)) + .is_err() + ); + drop(proof); + let result = block_on(repository.remove( + &fixture.execution, + &CheckoutActivity, + &managed, + independent, + )); + assert_eq!(result.is_err(), independent); + if !independent { + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, false)) + .unwrap(); + let archived = creation.with_file_name(format!("{}.removed", managed.id)); + assert_eq!(std::fs::read(&archived).unwrap(), creation_bytes); + assert!( + block_on(repository.list(&fixture.execution)) + .unwrap() + .is_empty() + ); + let fresh = block_on(repository.recreate_removed( + &fixture.execution, + &CheckoutActivity, + &Settings::default(), + &fixture.data, + "prj_test", + &managed, + )) + .unwrap(); + assert_ne!(fresh.id, managed.id); + assert!( + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, true)) + .is_err() + ); + assert_eq!( + std::fs::read_to_string(fresh.path.join("tracked.txt")).unwrap(), + "base" + ); + assert_eq!(std::fs::read(archived).unwrap(), creation_bytes); + } + assert_eq!(std::fs::read(path).unwrap(), original); + } +} + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn exact_snapshot_reacquisition_preserves_unknown_state_and_refuses_live_foreign_and_replaced_evidence() diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 4be4b4ee0..b27349a04 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -36,6 +36,33 @@ pub trait SnapshotLocationGuard: Send { fn bindings(&self) -> Result, String> { Ok(vec![]) } + /// Read-only verification of a proposed receipt against existing native + /// acknowledgements. Retain any existing marker through the consuming commit. + fn prepare_preparation_acknowledgement( + &mut self, + receipt: &super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + let resources = self.bindings()?; + if receipt.lifetime.owner.worktree_id.is_some() || !resources.is_empty() { + return Err("Native snapshot checkout acknowledgement is unsupported".into()); + } + if receipt.checkout_resources_digest.as_ref() + != Some(&receipt.lifetime.checkout_resources_digest(&resources)?) + { + return Err("Receipt differs from retained snapshot resources".into()); + } + Ok(()) + } + + /// Explicit native metadata publication after the exact consuming commit. + /// This must never reenter the primary Store or dispatch source effects. + fn acknowledge_preparation( + &mut self, + receipt: &super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + self.prepare_preparation_acknowledgement(receipt) + } } struct UnmanagedSnapshotLocation { _owner: Box, diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs index 693f1d42b..26fc59ba1 100644 --- a/crates/cyber-server/src/runtime/snapshot_lifetime.rs +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -169,6 +169,14 @@ pub trait SnapshotLifetime: Send { fn verify_preparation(&self, _origin: Option<&super::SnapshotOrigin>) -> Result<(), String> { Err("Native preparation phase proof is unavailable".into()) } + /// Read-only candidate check against retained native preparation receipts. + /// Called at the primary writer; no primary Store reentry or metadata writes. + fn prepare_preparation_acknowledgement( + &self, + _receipt: &super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + Err("Native preparation receipt verification is unavailable".into()) + } /// Metadata publication only, after observed durable preparation cancellation. fn acknowledge_preparation( &mut self, @@ -210,6 +218,26 @@ mod checkout_encoding_tests { binding.checkout_resources_digest(&[]).unwrap(), "a36ba5baced567a6dde73bc41c6e83c49017ba049b43fd59a5607b6204b5c5e7" ); + let mut receipt = super::super::SnapshotPreparationReceipt { + activity_id: "op_fixture".into(), + request_id: "cancel".into(), + confirmation: "a".repeat(64), + lifetime: binding.clone(), + native_sequence: 0, + session_sequence: 3, + origin: None, + source_admitted: Some(false), + checkout_resources_digest: Some(binding.checkout_resources_digest(&[]).unwrap()), + }; + assert_eq!( + receipt.checkout_acknowledgement_digest().unwrap(), + "9d475e87059bfef3f53d128c9be8029e36b8815289ad5a0cde8f7d5614938a4e" + ); + receipt.source_admitted = None; + assert!(receipt.checkout_acknowledgement_digest().is_err()); + receipt.source_admitted = Some(false); + receipt.checkout_resources_digest = None; + assert!(receipt.checkout_acknowledgement_digest().is_err()); let mut changed = binding.clone(); changed.file_identity.push_str("_replacement"); assert_ne!( diff --git a/crates/cyber-server/src/runtime/snapshot_preparation.rs b/crates/cyber-server/src/runtime/snapshot_preparation.rs index 864afd92b..f26a40515 100644 --- a/crates/cyber-server/src/runtime/snapshot_preparation.rs +++ b/crates/cyber-server/src/runtime/snapshot_preparation.rs @@ -52,6 +52,20 @@ pub struct SnapshotPreparationReceipt { impl SnapshotPreparationReceipt { pub const MAX_ENCODED_BYTES: u64 = 64 * 1024; + /// Issued by trusted adapters only after observing this exact committed pair. + pub fn checkout_acknowledgement_digest(&self) -> Result { + use sha2::{Digest, Sha256}; + validate_receipt(self).map_err(|error| error.to_string())?; + if self.checkout_resources_digest.is_none() || self.source_admitted != Some(false) { + return Err( + "Preparation receipt has no positive checkout acknowledgement authority".into(), + ); + } + self.lifetime.checkout_owner()?; + let bytes = serde_json::to_vec(&("snapshot-preparation-checkout-ack-v1", self)) + .map_err(|error| error.to_string())?; + Ok(format!("{:x}", Sha256::digest(bytes))) + } } struct Target { @@ -85,6 +99,7 @@ fn validate_receipt(receipt: &SnapshotPreparationReceipt) -> Result<(), StoreErr .is_some_and(|digest| { receipt.lifetime.version != 2 || receipt.lifetime.source_phase.is_none() + || receipt.source_admitted != Some(false) || digest.len() != 64 || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()) }) @@ -373,6 +388,17 @@ impl Worker { } Ok(()) } + fn acknowledge_native(&mut self, receipt: &SnapshotPreparationReceipt) -> Result<(), String> { + self.proof.acknowledge_preparation(receipt)?; + if receipt.checkout_resources_digest.is_some() { + self.verify_resources()?; + self.checkouts + .as_mut() + .ok_or("Missing original checkout acknowledgement proof")? + .acknowledge_preparation(receipt)?; + } + Ok(()) + } fn fingerprint(&self) -> Result { use sha2::{Digest, Sha256}; let bytes = serde_json::to_vec(&( @@ -466,15 +492,28 @@ impl Worker { // and its observed reply, rather than dropping them after the decision. let (_, (worker, receipt)) = store .append_pair_checked(owners, move |tx| { - self.authorize(tx)?; - Ok((events, (self, receipt))) + let mut worker = self; + if receipt.checkout_resources_digest.is_some() { + worker + .proof + .prepare_preparation_acknowledgement(&receipt) + .map_err(|error| invalid(&error))?; + worker + .checkouts + .as_mut() + .ok_or_else(|| invalid("Missing checkout receipt proof"))? + .prepare_preparation_acknowledgement(&receipt) + .map_err(|error| invalid(&error))?; + } + worker.authorize(tx)?; + Ok((events, (worker, receipt))) }) .map_err(preparation_error)?; let mut worker = worker; worker .handle .acknowledge_snapshot_uncertainty(&receipt.activity_id); - worker.proof.acknowledge_preparation(&receipt).map_err(|error| RuntimeError::Conflict(format!("Preparation cancellation committed; native journal acknowledgement requires reconciliation: {error}")))?; + worker.acknowledge_native(&receipt).map_err(|error| RuntimeError::Conflict(format!("Preparation cancellation committed; native journal or checkout acknowledgement requires reconciliation: {error}")))?; Ok::<_, RuntimeError>(receipt) }); let result = joined.await.map_err(|_| { @@ -703,8 +742,7 @@ impl Runtime { }) .map_err(preparation_error)?; worker - .proof - .acknowledge_preparation(&receipt) + .acknowledge_native(&receipt) .map_err(RuntimeError::Conflict)?; Ok(receipt) }) diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index d12696f1f..aec68b8e3 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -247,11 +247,11 @@ impl SnapshotWork<'_> { } impl SnapshotWorker { - fn checkout_proof( + fn checkout_resources( scope: &activity::Scope, - lease: super::LocationLease, + lease: &super::LocationLease, worktree: Option<&str>, - ) -> Result, RuntimeError> { + ) -> Result, RuntimeError> { let bindings = lease.snapshot_bindings().map_err(RuntimeError::Conflict)?; let expected = scope .snapshot_lifetime() @@ -277,6 +277,14 @@ impl SnapshotWorker { )); } } + Ok(bindings) + } + fn checkout_proof( + scope: &activity::Scope, + lease: super::LocationLease, + worktree: Option<&str>, + ) -> Result, RuntimeError> { + let bindings = Self::checkout_resources(scope, &lease, worktree)?; let proof = lease .settle_snapshot_retained() .map_err(RuntimeError::Invalid)?; @@ -473,13 +481,15 @@ impl Runtime { "Recovery native Location/creation binding changed".into(), )); } - let location = - SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; + let bindings = + SnapshotWorker::checkout_resources(&scope, &lease, state.info.worktree_id.as_deref())?; if let Some(lifetime) = &lifetime { lifetime - .bind_checkouts(&location.bindings().map_err(RuntimeError::Conflict)?) + .bind_checkouts(&bindings) .map_err(RuntimeError::Conflict)?; } + let location = + SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; Ok(SnapshotWorker { scope, admission, @@ -543,13 +553,15 @@ impl Runtime { .id .clone(); let worktree_id = state.info.worktree_id.clone(); - let location = - SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; + let bindings = + SnapshotWorker::checkout_resources(&scope, &lease, state.info.worktree_id.as_deref())?; if let Some(lifetime) = &lifetime { lifetime - .bind_checkouts(&location.bindings().map_err(RuntimeError::Conflict)?) + .bind_checkouts(&bindings) .map_err(RuntimeError::Conflict)?; } + let location = + SnapshotWorker::checkout_proof(&scope, lease, state.info.worktree_id.as_deref())?; Ok(SnapshotWorker { scope, runtime: self.clone(), diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index e787455f1..d9a465b6c 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -103,6 +103,8 @@ struct SnapshotClaims { claimed: std::sync::Mutex>, reacquisition: Option<(Arc, Arc)>, reacquire_stop: bool, + native_ack: Option>, + acquired_settlements: Option>, reacquire_inner: std::sync::Mutex>, } impl super::super::ToolHost for SnapshotClaims { @@ -143,10 +145,17 @@ impl super::super::ToolHost for SnapshotClaims { stop, })); } - Ok(Box::new(CheckoutProof { + let proof = CheckoutProof { calls: calls.clone(), fail_at: fail_at.clone(), - })) + }; + match &self.native_ack { + Some(state) => Ok(Box::new(ReceiptCheckoutProof { + proof, + state: state.clone(), + })), + None => Ok(Box::new(proof)), + } } fn claim_location<'a>( &'a self, @@ -169,7 +178,16 @@ impl super::super::ToolHost for SnapshotClaims { ) -> futures::future::BoxFuture<'a, Result> { Box::pin(async move { self.claimed.lock().unwrap().push(activity.into()); - Ok(super::super::LocationLease::unmanaged()) + match &self.acquired_settlements { + Some(settlements) => Ok(super::super::LocationLease::guarded( + None, + Box::new(BindingLease { + bindings: vec![], + settlements: settlements.clone(), + }), + )), + None => Ok(super::super::LocationLease::unmanaged()), + } }) } } @@ -244,6 +262,10 @@ impl super::super::LocationGuard for BindingLease { self.settlements.fetch_add(1, Ordering::SeqCst); Ok(()) } + fn settle_retained(self: Box) -> Result, String> { + self.settlements.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(())) + } fn snapshot_bindings(&self) -> Result, String> { Ok(self.bindings.clone()) } @@ -353,6 +375,43 @@ struct CheckoutProof { fail_at: Arc, } +#[derive(Default)] +struct NativeAckFixture { + attempts: AtomicUsize, + refuse_prepare: std::sync::atomic::AtomicBool, + refuse_publish: std::sync::atomic::AtomicBool, +} +struct ReceiptCheckoutProof { + proof: CheckoutProof, + state: Arc, +} +impl super::super::SnapshotLocationGuard for ReceiptCheckoutProof { + fn verify(&self) -> Result<(), String> { + self.proof.verify() + } + fn prepare_preparation_acknowledgement( + &mut self, + receipt: &super::super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + if self.state.refuse_prepare.load(Ordering::SeqCst) { + return Err("foreign existing native receipt".into()); + } + Ok(()) + } + fn acknowledge_preparation( + &mut self, + receipt: &super::super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + self.state.attempts.fetch_add(1, Ordering::SeqCst); + if self.state.refuse_publish.load(Ordering::SeqCst) { + return Err("native acknowledgement publication cut".into()); + } + Ok(()) + } +} + struct StopCheckoutProof { calls: Arc, stop_at: Arc, @@ -576,6 +635,12 @@ impl super::super::SnapshotLifetime for LifetimePermit { fn verify_preparation(&self, _: Option<&SnapshotOrigin>) -> Result<(), String> { Ok(()) } + fn prepare_preparation_acknowledgement( + &self, + _: &super::super::SnapshotPreparationReceipt, + ) -> Result<(), String> { + Ok(()) // Instrumented journal ownership; actual native receipts are tested in the backend. + } fn acknowledge_preparation( &mut self, _: &super::super::SnapshotPreparationReceipt, @@ -1322,6 +1387,100 @@ async fn preparation_consuming_writer_revalidates_original_checkout_proof_and_ro } } +#[tokio::test] +async fn preparation_native_ack_is_post_commit_and_only_explicit_reconciliation_retries_publication() + { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let native = Arc::new(NativeAckFixture::default()); + let host = Arc::new(SnapshotClaims { + reacquisition: Some(( + Arc::new(AtomicUsize::new(0)), + Arc::new(AtomicUsize::new(usize::MAX)), + )), + native_ack: Some(native.clone()), + ..Default::default() + }); + let runtime = runtime_with_snapshot_tools( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + reject_resources: false, + resources: Default::default(), + }), + host, + ); + let id = crash_unstarted(worker(&runtime, &session).await); + let review = runtime + .snapshot_preparation_review(&session, &id) + .await + .unwrap(); + let confirmation = super::super::SnapshotPreparationConfirmation { + confirmation: review.confirmation, + request_id: "native-ack-cut".into(), + }; + native.refuse_prepare.store(true, Ordering::SeqCst); + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .is_err() + ); + assert!( + runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap() + .is_none() + ); + assert_eq!(native.attempts.load(Ordering::SeqCst), 0); + native.refuse_prepare.store(false, Ordering::SeqCst); + native.refuse_publish.store(true, Ordering::SeqCst); + assert!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .is_err() + ); + let receipt = runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap() + .unwrap(); + assert_eq!(latest(&runtime, &session)["status"], "settled"); + assert_eq!(native.attempts.load(Ordering::SeqCst), 1); + assert_eq!( + runtime + .snapshot_preparation_confirm(&session, &id, &confirmation) + .await + .unwrap(), + receipt + ); + assert_eq!(native.attempts.load(Ordering::SeqCst), 1); + native.refuse_publish.store(false, Ordering::SeqCst); + assert_eq!( + runtime + .snapshot_preparation_acknowledge(&session, &id, &confirmation.request_id) + .await + .unwrap(), + receipt + ); + assert_eq!(native.attempts.load(Ordering::SeqCst), 2); + assert_eq!( + runtime + .snapshot_preparation_lookup(&session, &id, &confirmation.request_id) + .await + .unwrap(), + Some(receipt) + ); + assert_eq!(native.attempts.load(Ordering::SeqCst), 2); + runtime.shutdown().await; + base.shutdown().await; +} + #[tokio::test] async fn preparation_receipt_binds_resources_and_lookup_never_reopens_changed_native_evidence() { let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; @@ -1387,6 +1546,39 @@ async fn preparation_receipt_binds_resources_and_lookup_never_reopens_changed_na base.shutdown().await; } +#[tokio::test] +async fn complete_resource_publication_precedes_any_checkout_admission_settlement() { + for reject in [false, true] { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let settlements = Arc::new(AtomicUsize::new(0)); + let runtime = runtime_with_snapshot_tools( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + reject_resources: reject, + resources: Default::default(), + }), + Arc::new(SnapshotClaims { + acquired_settlements: Some(settlements.clone()), + ..Default::default() + }), + ); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let result = runtime.snapshot_worker(&handle, &state).await; + assert_eq!(result.is_err(), reject); + assert_eq!(settlements.load(Ordering::SeqCst), usize::from(!reject)); + drop(result); + drop(state); + runtime.shutdown().await; + base.shutdown().await; + } +} + #[tokio::test] async fn resource_publication_failure_retains_unknown_reservation_without_origin_or_source_work() { let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index de0d0bd01..5711ffb0f 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -119,6 +119,32 @@ impl SnapshotLifetime for Lifetime { .ok_or("Missing source-admission namespace")?; store.record_source_admission(&self.binding, origin) } + fn prepare_preparation_acknowledgement( + &self, + receipt: &cyber_server::runtime::SnapshotPreparationReceipt, + ) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + if receipt.lifetime != self.binding { + return Err("Foreign preparation receipt candidate".into()); + } + self.verify_preparation(receipt.origin.as_ref())?; + if let Some(origin) = &receipt.origin { + let store = Store::existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Missing preparation receipt namespace")?; + if store + .preparation_cancellation(&origin.binding.journal_id)? + .as_ref() + .is_some_and(|existing| existing != receipt) + { + return Err("Journal already has another preparation cancellation receipt".into()); + } + } + Ok(()) + } fn acknowledge_preparation( &mut self, receipt: &cyber_server::runtime::SnapshotPreparationReceipt, diff --git a/crates/cyber-snapshot/src/copy/preparation.rs b/crates/cyber-snapshot/src/copy/preparation.rs index b50b050b9..07ecdacf9 100644 --- a/crates/cyber-snapshot/src/copy/preparation.rs +++ b/crates/cyber-snapshot/src/copy/preparation.rs @@ -221,7 +221,20 @@ mod tests { assert!(directory.symlink_metadata(FILE).is_err()); receipt.checkout_resources_digest = Some(receipt.lifetime.checkout_resources_digest(&[]).unwrap()); + lifetime + .prepare_preparation_acknowledgement(&receipt) + .unwrap(); lifetime.acknowledge_preparation(&receipt).unwrap(); + let mut different = receipt.clone(); + different.request_id.push_str("_foreign"); + assert!( + lifetime + .prepare_preparation_acknowledgement(&different) + .is_err() + ); + lifetime + .prepare_preparation_acknowledgement(&receipt) + .unwrap(); let recorded: SnapshotPreparationReceipt = serde_json::from_slice(&storage::read(&directory, FILE, LIMIT).unwrap()).unwrap(); assert_eq!(recorded, receipt); diff --git a/crates/cyber-store/migrations/20261107020000_snapshot_checkout_acknowledgements.sql b/crates/cyber-store/migrations/20261107020000_snapshot_checkout_acknowledgements.sql new file mode 100644 index 000000000..ef4d99af2 --- /dev/null +++ b/crates/cyber-store/migrations/20261107020000_snapshot_checkout_acknowledgements.sql @@ -0,0 +1,3 @@ +-- Native checkout disposition binds the exact durable consuming receipt. +-- Earlier writers must preserve the new marker and its retained construction evidence. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index 986ea600d..77c87e1e9 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -197,6 +197,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261107010000_snapshot_consumed_resources", sql: include_str!("../migrations/20261107010000_snapshot_consumed_resources.sql"), }, + Migration { + id: "20261107020000_snapshot_checkout_acknowledgements", + sql: include_str!("../migrations/20261107020000_snapshot_checkout_acknowledgements.sql"), + }, ]; pub(crate) enum Outcome { @@ -508,7 +512,13 @@ mod tests { let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { panic!("previous writer accepted consumed checkout evidence"); }; - assert_eq!(unknown, ["20261107010000_snapshot_consumed_resources"]); + assert_eq!( + unknown, + MIGRATIONS[at..] + .iter() + .map(|migration| migration.id) + .collect::>() + ); let retained: String = db .query_row( "SELECT data FROM event WHERE id='evt_resources'", @@ -519,6 +529,36 @@ mod tests { assert_eq!(retained, evidence); } + #[test] + fn native_acknowledgement_catalog_preserves_unknown_claim_and_disposition_bytes() { + let temp = tempfile::tempdir().unwrap(); + let claim = temp.path().join("op_original.lock"); + let marker = temp.path().join("op_original.snapshot-ack"); + std::fs::write(&claim, b"retained unknown claim").unwrap(); + std::fs::write(&marker, b"retained exact receipt metadata").unwrap(); + let mut db = Connection::open_in_memory().unwrap(); + let at = MIGRATIONS + .iter() + .position(|migration| { + migration.id == "20261107020000_snapshot_checkout_acknowledgements" + }) + .unwrap(); + migrate(&mut db, &MIGRATIONS[..at]).unwrap(); + migrate(&mut db, MIGRATIONS).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { + panic!("previous writer accepted native checkout disposition"); + }; + assert_eq!( + unknown, + ["20261107020000_snapshot_checkout_acknowledgements"] + ); + assert_eq!(std::fs::read(claim).unwrap(), b"retained unknown claim"); + assert_eq!( + std::fs::read(marker).unwrap(), + b"retained exact receipt metadata" + ); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index f26ff0f7d..346d7f81c 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -1323,6 +1323,48 @@ impl cyber_server::runtime::SnapshotLocationGuard for ReacquiredSnapshotProof { self.verify()?; Ok(self.0.iter().map(|claim| claim.binding().clone()).collect()) } + fn prepare_preparation_acknowledgement( + &mut self, + receipt: &cyber_server::runtime::SnapshotPreparationReceipt, + ) -> Result<(), String> { + let resources = self.bindings()?; + let digest = receipt.lifetime.checkout_resources_digest(&resources)?; + if receipt.checkout_resources_digest.as_ref() != Some(&digest) { + return Err("Committed receipt differs from original checkout resources".into()); + } + let receipt_digest = receipt.checkout_acknowledgement_digest()?; + for claim in &mut self.0 { + cyber_core::worktrees::CheckoutAcknowledgement::new( + claim.binding(), + &digest, + &receipt_digest, + ) + .and_then(|ack| ack.retain_existing(claim)) + .map_err(|error| error.to_string())?; + } + self.verify() + } + fn acknowledge_preparation( + &mut self, + receipt: &cyber_server::runtime::SnapshotPreparationReceipt, + ) -> Result<(), String> { + let resources = self.bindings()?; + let digest = receipt.lifetime.checkout_resources_digest(&resources)?; + if receipt.checkout_resources_digest.as_ref() != Some(&digest) { + return Err("Committed receipt differs from original checkout resources".into()); + } + let receipt_digest = receipt.checkout_acknowledgement_digest()?; + for claim in &mut self.0 { + cyber_core::worktrees::CheckoutAcknowledgement::new( + claim.binding(), + &digest, + &receipt_digest, + ) + .and_then(|ack| ack.publish(claim)) + .map_err(|error| error.to_string())?; + } + self.verify() + } } struct SnapshotWorktreeProof(Vec); diff --git a/crates/cyber-tools/tests/worktree_setup.rs b/crates/cyber-tools/tests/worktree_setup.rs index 1b2e6f74c..8d93ce478 100644 --- a/crates/cyber-tools/tests/worktree_setup.rs +++ b/crates/cyber-tools/tests/worktree_setup.rs @@ -2014,7 +2014,7 @@ async fn lifetime_bound_checkout_host_preserves_context_and_refuses_foreign_requ serde_json::from_slice::(&retained).unwrap()["settled"], false ); - let proof = flow + let mut proof = flow .f .host .reacquire_snapshot_location(&info, &exited, &acquired) @@ -2041,6 +2041,40 @@ async fn lifetime_bound_checkout_host_preserves_context_and_refuses_foreign_requ .reacquire_snapshot_location(&info, &foreign, &acquired) .is_err() ); + // Adapter fixture: the runtime suites attest the actual committed pair. + let receipt = cyber_server::runtime::SnapshotPreparationReceipt { + activity_id: exited.owner.activity_id.clone(), + request_id: "fixture-cancel".into(), + confirmation: "a".repeat(64), + lifetime: exited.clone(), + native_sequence: 0, + session_sequence: 8, + origin: None, + source_admitted: Some(false), + checkout_resources_digest: Some(exited.checkout_resources_digest(&acquired).unwrap()), + }; + let marker = directory.join("op_exited.snapshot-ack"); + proof.prepare_preparation_acknowledgement(&receipt).unwrap(); + assert!(!marker.exists()); + proof.acknowledge_preparation(&receipt).unwrap(); + let acknowledged = std::fs::read(&marker).unwrap(); + proof.acknowledge_preparation(&receipt).unwrap(); + assert_eq!(std::fs::read(&marker).unwrap(), acknowledged); + let native: serde_json::Value = serde_json::from_slice(&acknowledged).unwrap(); + assert_eq!( + native["receipt_digest"], + receipt.checkout_acknowledgement_digest().unwrap() + ); + let mut different = receipt.clone(); + different.request_id = "different-request".into(); + assert!( + proof + .prepare_preparation_acknowledgement(&different) + .is_err() + ); + let mut legacy = receipt.clone(); + legacy.checkout_resources_digest = None; + assert!(proof.acknowledge_preparation(&legacy).is_err()); drop(proof); assert_eq!( std::fs::read(directory.join("op_exited.lock")).unwrap(), diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 6ded8ebe2..bf2d00e5d 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -196,3 +196,14 @@ journal metadata reconciliation checks the committed resource binding. Original native checkout acknowledgement, partial-constructor recovery, Git journaling and reference-aware cleanup remain required before the complete recovery increment is reviewable for delivery. + + +The pre-effect native checkout acknowledgement continuation publishes an exact +receipt-bound disposition while preserving original claim bytes and independent +unknown fences. Complete resource publication now precedes checkout admission +settlement. Candidate journal/checkout receipts are checked before the paired +commit, and interrupted publication resumes only through explicit metadata +reconciliation. Acknowledged creation metadata is retained across authorized +worktree removal/recreation. Partial acquisition/corrupt construction recovery, +post-effect/owned Git integration, reference-aware reclamation and full native/ +client acceptance remain open; this does not complete the recovery delivery increment. diff --git a/docs/rewind.md b/docs/rewind.md index ea02ae9b7..9db4a048f 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -278,13 +278,14 @@ lookup return the original durable pair without reopening native proof. After a committed receipt with missing native acknowledgement, POST `/sessions//rewind/preparations//requests//acknowledge` to reconcile metadata under the exact proof. This never replays source changes. -Checkout/process uncertainty remains independent. Cancelling an unstarted -keep/rollback request leaves its original failed restore unresolved; its lookup +A receipt with a complete acquired checkout binding can reconcile only that +preparation's exact exited checkout claims. Independent checkout and process +uncertainty remains fenced. Cancelling an unstarted keep/rollback request leaves its original failed restore unresolved; its lookup reports `cancelled_before_origin` or `cancelled_before_effects`. Created source directories, private payloads, native phase records and lifetime preparations remain protected for acknowledged safe cleanup. Do not edit records -or remove evidence to force a retry. Independent managed-checkout proof, Git +or remove evidence to force a retry. Partial acquisition recovery, owned Git journaling, reclamation, typed recovery clients and broader native crash/platform acceptance remain unfinished. @@ -293,22 +294,35 @@ Managed checkout snapshots use an independent claim named by the snapshot's nati activity, so they do not reuse a Session, binary document, MCP or language-server claim. The native claim remains locked through worker settlement. Hosts without explicit support refuse managed snapshot admission. A crash while acquiring a claim -can still leave unknown checkout evidence: preparation cancellation does not clear -that evidence or authorize checkout removal. Exact native claim reconciliation is -still required for that acquisition cut; retain the evidence for recovery. +can still leave unknown checkout evidence. When the complete acquired set was +published before checkout settlement, its committed preparation receipt can +acknowledge those exact claims while retaining their original bytes. Missing or +incomplete publication and legacy claims keep their fences; retain that evidence +for the remaining recovery path. While a managed snapshot is running, the worker verifies its pinned checkout claim and worktree creation metadata before source changes and consuming commits. Aliased or replaced claim files, changed creation contents and replaced activity directories -refuse. These checks preserve evidence; they do not grant permission to clear an -unknown checkout claim after interrupted acquisition. +refuse. These checks preserve evidence. New checkout acknowledgement also requires +the exact committed preparation receipt and complete acquired-resource binding; +verification alone cannot clear an unknown claim. New Linux/macOS lifetime-bound checkout claims publish their complete native record before becoming visible under the activity name. A pre-publication crash retains a construction record and prevents reuse of that activity. Runtime validates Session -creation, directory, lifetime digest and the whole returned resource set before -settling native ownership. Construction evidence is retained for the pending safe +creation, directory, lifetime digest and the whole returned resource set, then +persists that set before settling checkout admission. Construction evidence is retained for the pending safe recovery/cleanup path; it cannot be removed to force a retry. Windows does not yet admit the lifetime-bound durability path. + +Native preparation acknowledgement publishes separate metadata after the consuming +commit. It never rewrites an original checkout claim or clears another Session's +unknown activity. If acknowledgement is interrupted, look up the committed request +and use its explicit acknowledgement endpoint; confirming again only returns the +receipt. Existing native journal or checkout receipts must match before a new +consuming commit, so a database backup cannot replace a different native receipt. +An authorized later worktree removal retains referenced creation metadata for +retry/recreation. Payloads, construction records and archived creation evidence +remain protected until reference-aware cleanup is implemented. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index ced8d4108..21b6913fb 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1546,3 +1546,106 @@ Stop fix and assertion-helper extraction, including pre-origin and published-ori service kills, restart/lookup, and partial keep/rollback. Their successful receipt assertions require review/commit resource agreement and the exact complete empty-set digest after published origins. No additional parent acceptance is claimed. + + +## Native acknowledgement of exact pre-effect checkout claims (2026-11-07) + +The production retained checkout proof can now publish a separate native +acknowledgement after observed durable preparation cancellation. The marker binds +the exact version-2 claim, the complete acquired-resource digest and a versioned +SHA256 digest of the entire committed receipt. The fixed known-answer encoding +refuses digest-less or non-positive pre-effect receipts. No original claim bytes +are reset, rewritten or deleted. `SnapshotClaim` still exposes read-only proof; +the separate `CheckoutAcknowledgement` adapter authority performs this specific +metadata publication and cannot operate on a different or legacy claim. + +Publication retains the original claim lock and pins. A complete bounded record +with its own native file identity and construction name is staged, flushed, +published without replacement and directory-synchronized. Existing matching markers +are revalidated, retained and synchronized on retry. Replaced/aliased markers, +foreign contexts and different receipts refuse. Marker encoding is bounded to +32 KiB and shares the 4096-entry activity inventory. Native bound publication is +admitted only on Linux/macOS; unsupported durability refuses before creating +metadata. All construction evidence remains retained for the pending cleanup path. + +Before paired commit the actual worker checks existing native journal cancellation +metadata and retains any existing checkout marker against the proposed receipt. +This prevents a database backup from producing a conflicting receipt even if a +prior acknowledgement stopped between journal and checkout publication. The writer +then rechecks native resources, current authority and Stop. Only the observed commit +reply permits journal/checkout publication. A native publication error keeps the +committed pair and reports reconciliation. Confirm/lookup retries remain database- +only; explicit acknowledgement reacquires and attests the same proof before retrying +metadata. Source and transcript are never replayed by those operations. + +Complete resource publication now precedes checkout admission settlement. Publication +failure does not settle any acquired checkout. A cut after complete publication and +before settlement retains the positive resource set needed for exact pre-effect +claim reconciliation. Partial acquisition, incomplete publication and corrupt claim +records still refuse; this step does not invent missing provenance or complete all +constructor cuts. Normal admitted-source/Git recovery and their consuming native +checkout integration remain separate required work. + +Removal checks every independent activity record under its original lock. An exact +validated native acknowledgement can discharge only its snapshot claim; legacy or +independent unknown Session/process records remain fenced even with --force. Only +acknowledgement-construction names tied to that exact acknowledged owner are pinned +as retained metadata; they cannot authorize anything alone and are not deleted. +Creation records referenced by native acknowledgement are archived by a no-replace +rename in their original creation directory during an authorized worktree removal. +Their native file identity and bytes survive removal retry/recreation. Removal-only +verification may use that exact archived record when the live route is absent; +ordinary snapshot worker reacquisition still requires the live creation route. A +new incarnation's live creation never falls back to old archived proof. + +Actual core regressions exercise marker idempotence, original/independent unknown +byte preservation, native replacement and hard-link refusal, foreign/unacknowledged +stages and a changed Managed context. A real process is killed after durable stage +preparation and before publication; retry completes without rewriting the claim or +deleting the retained stage, which remains locked during later removal admission. +Namespace replacement immediately before publication retains the original stage and +installs no marker in the replacement. Real Git tests prove removal after exact +acknowledgement, independent unknown refusal, durable creation archive, repeated +removal, clean recreation and refusal to affect the new incarnation. Production-host +coverage exercises actual native claims with an explicitly instrumented lifetime/ +receipt: it is adapter evidence, not a full owned-Git runtime acceptance claim. + +Runtime instrumentation refuses an existing foreign native receipt before commit, +then fails publication after the commit. Lookup and confirm retries return the +committed pair without another attempt; explicit acknowledgement alone retries. +It also proves publication failure precedes any checkout admission settlement. +Native backend coverage refuses a different journal receipt before consuming commit +and preserves identical retry, resource-digest and legacy journal-only behavior. + +The code-review skill is applied across every changed core/native, runtime/port, +Store compatibility, production adapter, test and documentation file. Review checks +no source/process work in metadata callbacks, no primary Store reentry, candidate +receipt checks before commit, lifetime/claim/marker lock retention, exact native +route/content checks, bounded no-replace publication, retry durability, orphan/alias +refusal, independent unknown preservation and removal/recreation compatibility. +Review identified the creation-record lifetime and publication-before-settlement +ordering gaps, both addressed with regression coverage. Partial/corrupt constructor +cuts, post-effect/owned Git integration, reference/reader/lease-aware reclamation, +complete clients and full native/platform acceptance remain required with M2–M8 and +2.4/3.2–3.4. No parent task is accepted and the full recovery increment is not yet +ready for delivery. + + +Verification on local macOS arm64: all 415 core unit tests passed with three ignored +tests, including the new acknowledgement subprocess fixture invoked by its parent +kill test. All 73 core worktree integration cases passed with one ignored fixture; +the native acknowledgement case includes real removal/retry/recreation. All 428 +server unit tests passed with one ignored fixture after the publication-ordering +fix; the earlier focused snapshot run passed 44 cases. All 85 snapshot cases passed +with one ignored fixture, all 64 Store tests passed, and all 40 production-host +worktree cases passed. All 16 real CLI branch/recovery cases passed again after the +final runtime ordering change. All-target Clippy for core/server/tools/Store/ +snapshot/App/CLI passed with warnings denied. Formatting, diff and generated SDK +consistency passed. Strict pinned OpenSpec passed all 117 items and spec lint reports +0 errors/25 warnings. SDK execution tests, full-workspace tests, native Linux/Windows +execution and live TUI/App recovery acceptance were not run for this increment. +Native sandbox deprecation and Darwin linker warnings were observed without an +unchanged-main comparison or origin attribution. An initial integration compile +used the async listing API incorrectly; it was corrected before the passing full +worktree run. No critical/major review issue remains in this pre-effect native +acknowledgement scope. The broader unfinished obligations above remain required. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 33e667a80..43f98e58c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -13,8 +13,9 @@ durable request reservation and paired native settlement. Local real-service partial-restore kills exercise keep/rollback and lookup without replay. Explicit journal acknowledgement reconciliation is implemented for new file-copy receipts; pre-effect cancellation is available for new version-2 lifetime-bound file-copy -workers, including a published origin with no admitted source effects. Independent -resource cuts, Git journaling, +workers, including a published origin with no admitted source effects. Complete +acquired version-2 checkout sets now support separate receipt-bound native +acknowledgement. Partial/corrupt constructor cuts, post-effect/owned Git integration, reclamation, complete clients and the platform matrix remain unfinished. Parent tasks are open; the complete recovery delivery is not merge-ready. @@ -402,3 +403,48 @@ receipts lacking the digest remain readable but cannot acquire new checkout acknowledgement authority. Request lookup remains database-only. Explicit metadata reconciliation must match the committed digest before accessing original claims. This binding does not by itself acknowledge an unknown native checkout claim. + + +### Native acknowledgement of consumed checkout claims + +After observed durable preparation cancellation, the retained original checkout +proof may publish a separate bounded acknowledgement record. It binds the exact +version-2 claim, the consumed complete-resource digest and a versioned digest of +the complete committed preparation receipt. Original claim bytes are never reset +or rewritten. Publication holds the original lock, stages and flushes a complete +private record, publishes without replacement and synchronizes the directory. +The writer also checks existing journal cancellation metadata and retains any +existing checkout marker against the proposed receipt +before commit; a database backup cannot replace a different native receipt. +The acknowledgement retains its own native file identity and construction name; +a replacement, alias, changed context or different receipt refuses. + +Removal admission may recognize that exact acknowledged claim while holding its +original lock and native pins. It still checks every independent Session/process/ +checkout record. A partial acknowledgement stage never supplies authority. A +retained stage with the exact new acknowledgement-construction name is metadata +only after a matching stable acknowledgement exists; removal pins it without +deleting it. Orphan/foreign stages, unacknowledged claim construction, legacy claims +and unknown records retain their fences. All payload and construction reclamation +remains subject to the separate reference/reader/lease-aware cleanup protocol. +Retry lookup never publishes metadata. Explicit acknowledgement reconciliation +reacquires the original lifetime/claims, attests the committed pair and resource +digest, then idempotently completes journal and checkout metadata. A crash during +one resource acknowledgement leaves completed markers and retained stages, and the +same receipt can resume without repeating source effects. Unsupported lifetime-bound +native directory durability refuses before metadata creation. + +A later authorized managed-worktree removal retains the original creation record +under its creation ID when native checkout acknowledgements reference it. The +no-replace archival rename preserves its native file identity and bytes. Removal +retries may verify that archived record through the same retained creation-directory +identity; ordinary worker reacquisition still requires the live creation route. +This preserves acknowledgement proof across removal completion/recreation without +relaxing any independent unknown claim or reclaiming referenced creation evidence. + +Complete resource publication precedes checkout admission settlement as well as +origin/source dispatch. A publication failure leaves every acquired claim +unsettled. A cut after complete publication and before checkout settlement has +positive exited-lifetime/resource evidence for the new metadata acknowledgement; +partial acquisition and corrupt/incomplete records remain fenced until their +separate exact recovery proof is implemented. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index a6f030890..a12645f0f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -122,6 +122,14 @@ - **AND** original claim locks and pins SHALL remain held through consuming commit and observed reply, with changed or live evidence refusing - **AND** historical receipts without this binding SHALL gain no new checkout acknowledgement authority, while request lookup SHALL remain database-only +#### Scenario: Durable preparation receipt acknowledges only its native checkout claims +- **WHEN** the consuming preparation commit is observed durably acknowledged under retained original locks +- **THEN** native checkout acknowledgement SHALL bind the exact version-2 claim, complete-resource digest and committed receipt digest without rewriting original claim bytes +- **AND** no-replace durable publication and exact native marker identity SHALL reject changed, aliased, foreign or different-receipt evidence +- **AND** removal SHALL continue to refuse independent unknown claims and incomplete or foreign construction evidence +- **AND** a retained acknowledgement stage SHALL grant no authority by itself and SHALL be preserved until separate acknowledged reference-aware reclamation +- **AND** only explicit metadata reconciliation after committed receipt attestation SHALL resume interrupted acknowledgements without source replay + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index eccad53e9..fa28cc196 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -234,3 +234,18 @@ of native evidence; explicit journal metadata acknowledgement verifies the commi digest. Historical digest-less receipts retain only their existing journal authority. Native unknown-claim acknowledgement is still required: this continuation does not settle a checkout claim, delete construction evidence or accept any parent task. + + +Pre-effect native checkout acknowledgement continuation: observed paired receipts +with complete version-2 resource bindings now publish separate exact native markers +under retained original locks. Original claim bytes and independent unknown claims +are preserved. Read-only candidate checks at the writer reject conflicting existing +journal or checkout receipts; lookup never retries metadata. Explicit acknowledgement +reconciles interrupted journal/checkout publication without source replay. Verified +markers may admit later removal while pinning retained acknowledgement stages; +original creation records are archived without replacement when needed for removal +retry/recreation proof. Complete resource publication precedes checkout admission +settlement. Actual native pre-publication kill/namespace replacement and real Git +removal/recreation cases cover this step. M2–M8 and parent tasks remain open for +partial acquisition/corrupt construction, post-effect/owned Git integration, safe +reclamation, complete clients and full crash/platform acceptance. From 4378f4e6fc8279586c8996a8b0a5d56a641f9917 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 01:43:28 -0400 Subject: [PATCH 19/87] Verify exact Git snapshot paths and blob object identities --- Cargo.lock | 1 + crates/cyber-snapshot/Cargo.toml | 1 + crates/cyber-snapshot/src/git.rs | 22 ++-- crates/cyber-snapshot/src/git/images.rs | 112 ++++++++++++++++++ crates/cyber-snapshot/tests/snapshots.rs | 89 ++++++++++++++ docs/rewind.md | 5 + .../implementation.md | 39 ++++++ .../managed-recovery.md | 11 ++ .../specs/snapshots-checkpoints/spec.md | 5 + .../complete-daily-coding-surfaces/tasks.md | 9 ++ 10 files changed, 280 insertions(+), 14 deletions(-) create mode 100644 crates/cyber-snapshot/src/git/images.rs diff --git a/Cargo.lock b/Cargo.lock index 6913fe6a7..0f5cd9627 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1311,6 +1311,7 @@ dependencies = [ "serde", "serde_json", "sha1", + "sha2 0.10.9", "similar 3.2.0", "tempfile", "tokio", diff --git a/crates/cyber-snapshot/Cargo.toml b/crates/cyber-snapshot/Cargo.toml index 64203caeb..9b3c6d531 100644 --- a/crates/cyber-snapshot/Cargo.toml +++ b/crates/cyber-snapshot/Cargo.toml @@ -13,6 +13,7 @@ cyber-server = { path = "../cyber-server" } futures = "0.3" serde_json.workspace = true sha1 = "0.10" +sha2.workspace = true serde.workspace = true blake3 = "1" diffy = "0.4" diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 4e36c6688..8a7a279ef 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -7,6 +7,7 @@ use cyber_server::runtime::{FileDiff, Snapshot}; use sha1::{Digest, Sha1}; #[path = "git_history.rs"] pub(crate) mod history; +mod images; /// A shadow git directory bound to one worktree and scoped to one Location. #[derive(Debug, Clone)] @@ -399,22 +400,15 @@ impl Shadow { /// `(mode, contents)` of a path in a tree, or `None` when absent. pub async fn blob(&self, tree: &str, path: &str) -> Result)>, String> { self.recorded_exclusions(tree).await?; - let listing = self.run(&["ls-tree", "-z", tree, "--", path]).await?; - let entry = String::from_utf8_lossy(&listing).to_string(); - let Some(meta) = entry.split('\t').next().filter(|m| !m.is_empty()) else { - return Ok(None); - }; - let fields: Vec<&str> = meta.split(' ').collect(); - let [mode, kind, sha] = fields[..] else { + let listing = self + .run(&["--literal-pathspecs", "ls-tree", "-z", tree, "--", path]) + .await?; + let Some((mode, object)) = images::entry(&listing, path)? else { return Ok(None); }; - if kind != "blob" { - return Ok(None); - } - Ok(Some(( - mode.to_string(), - self.run(&["cat-file", "blob", sha]).await?, - ))) + let bytes = self.run(&["cat-file", "blob", &object]).await?; + images::verify(&object, &bytes)?; + Ok(Some((mode, bytes))) } } diff --git a/crates/cyber-snapshot/src/git/images.rs b/crates/cyber-snapshot/src/git/images.rs new file mode 100644 index 000000000..038c8f462 --- /dev/null +++ b/crates/cyber-snapshot/src/git/images.rs @@ -0,0 +1,112 @@ +//! Exact immutable images, shared by Git restore and owned-journal materialization. +use sha1::{Digest, Sha1}; +use sha2::Sha256; + +pub(super) fn entry(listing: &[u8], path: &str) -> Result, String> { + if path.is_empty() + || path.len() > 4096 + || path.contains('\0') + || std::path::Path::new(path) + .components() + .any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err("Invalid Git image path".into()); + } + if listing.is_empty() { + return Ok(None); + } + let record = listing + .strip_suffix(&[0]) + .ok_or("Unterminated Git image listing")?; + if record.contains(&0) { + return Err("Ambiguous Git image listing".into()); + } + let split = record + .iter() + .position(|byte| *byte == b'\t') + .ok_or("Malformed Git image listing")?; + if &record[split + 1..] != path.as_bytes() { + return Err("Git image path differs from the requested literal path".into()); + } + let metadata = + std::str::from_utf8(&record[..split]).map_err(|_| "Invalid Git image metadata")?; + let fields: Vec<_> = metadata.split(' ').collect(); + if let ["040000", "tree", object] = fields.as_slice() { + if !oid(object) { + return Err("Invalid Git directory image identity".into()); + } + return Ok(None); // A directory is not a file image at this exact path. + } + let [mode, "blob", object] = fields.as_slice() else { + return Err("Unsupported Git image entry".into()); + }; + if !matches!(*mode, "100644" | "100755" | "120000") || !oid(object) { + return Err("Invalid Git image mode or object identity".into()); + } + Ok(Some(((*mode).into(), (*object).into()))) +} +fn oid(object: &str) -> bool { + matches!(object.len(), 40 | 64) + && object + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} +pub(super) fn verify(object: &str, bytes: &[u8]) -> Result<(), String> { + let header = format!("blob {}\0", bytes.len()); + let actual = match object.len() { + 40 => { + let mut hash = Sha1::new(); + hash.update(header.as_bytes()); + hash.update(bytes); + format!("{:x}", hash.finalize()) + } + 64 => { + let mut hash = Sha256::new(); + hash.update(header.as_bytes()); + hash.update(bytes); + format!("{:x}", hash.finalize()) + } + _ => return Err("Invalid Git image object identity".into()), + }; + if actual != object { + return Err("Git snapshot image does not match its object identity".into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn exact_nul_records_preserve_unusual_paths_and_refuse_ambiguous_or_foreign_images() { + let object = "a".repeat(40); + let path = ":(glob)*\tline\nname"; + let record = format!("100755 blob {object}\t{path}\0"); + assert_eq!( + entry(record.as_bytes(), path).unwrap(), + Some(("100755".into(), object.clone())) + ); + assert_eq!(entry(b"", path).unwrap(), None); + assert!(entry(b"", "../outside").is_err()); + assert_eq!( + entry(format!("040000 tree {object}\t{path}\0").as_bytes(), path).unwrap(), + None + ); + assert!(entry(record.as_bytes(), "different").is_err()); + assert!(entry(record.trim_end_matches('\0').as_bytes(), path).is_err()); + assert!(entry(format!("{record}{record}").as_bytes(), path).is_err()); + assert!(entry(format!("160000 commit {object}\t{path}\0").as_bytes(), path).is_err()); + assert!(entry(format!("100644 blob invalid\t{path}\0").as_bytes(), path).is_err()); + } + #[test] + fn git_object_hashes_are_checked_with_the_type_and_length_frame() { + // Independent known Git empty-blob identities for both repository formats. + verify("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", b"").unwrap(); + verify( + "473a0f4c3be8a93681a267e3b1e9a7dcda1185436fe141f7749120a303721813", + b"", + ) + .unwrap(); + assert!(verify("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", b"changed").is_err()); + } +} diff --git a/crates/cyber-snapshot/tests/snapshots.rs b/crates/cyber-snapshot/tests/snapshots.rs index e5bb6f85b..1fbd441c2 100644 --- a/crates/cyber-snapshot/tests/snapshots.rs +++ b/crates/cyber-snapshot/tests/snapshots.rs @@ -492,3 +492,92 @@ async fn ignored_build_directories_use_one_bounded_exclusion_prefix() { assert_eq!(evidence.len(), 1); assert!(evidence[0].ends_with(": target/")); } + +#[cfg(unix)] +#[tokio::test] +async fn git_restore_materializes_literal_unusual_paths_without_touching_the_user_index() { + let r = Repo::new(); + let names = ["[ab].txt", "a.txt", "b.txt", ":(glob)*", "tab\tline\nfile"]; + for (index, name) in names.iter().enumerate() { + std::fs::write(r.root.join(name), format!("before {index}\n")).unwrap(); + } + let index = std::fs::read(r.root.join(".git/index")).unwrap(); + let target = r.snaps.track(&r.dir()).await.unwrap().unwrap(); + for (item, name) in names.iter().enumerate() { + std::fs::write(r.root.join(name), format!("after {item}\n")).unwrap(); + } + let recorded = r.snaps.track(&r.dir()).await.unwrap().unwrap(); + r.snaps + .restore_reviewed( + &r.dir(), + &target.tree, + &recorded.tree, + &recorded.tree, + Arc::new(|| Ok(())), + ) + .await + .unwrap(); + for (item, name) in names.iter().enumerate() { + assert_eq!( + std::fs::read_to_string(r.root.join(name)).unwrap(), + format!("before {item}\n") + ); + } + assert_eq!(std::fs::read(r.root.join(".git/index")).unwrap(), index); +} + +#[cfg(unix)] +#[tokio::test] +async fn corrupt_git_snapshot_blob_refuses_restore_and_preserves_current_source() { + use std::os::unix::fs::PermissionsExt; + let r = Repo::new(); + std::fs::write(r.root.join("a.txt"), "unique original snapshot image").unwrap(); + let target = r.snaps.track(&r.dir()).await.unwrap().unwrap(); + std::fs::write(r.root.join("a.txt"), "replacement snapshot image").unwrap(); + let recorded = r.snaps.track(&r.dir()).await.unwrap().unwrap(); + let project = std::fs::read_dir(r.data.join("snapshot")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let shadow = std::fs::read_dir(project) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.join("HEAD").exists()) + .unwrap(); + let object = |snapshot: &str| { + let out = Command::new("git") + .arg("--git-dir") + .arg(&shadow) + .args(["rev-parse", &format!("{snapshot}:a.txt")]) + .output() + .unwrap(); + assert!(out.status.success()); + let id = String::from_utf8(out.stdout).unwrap(); + let id = id.trim(); + shadow.join("objects").join(&id[..2]).join(&id[2..]) + }; + let original = object(&target.tree); + let replacement = std::fs::read(object(&recorded.tree)).unwrap(); + std::fs::set_permissions(&original, std::fs::Permissions::from_mode(0o600)).unwrap(); + std::fs::write(&original, replacement).unwrap(); + let index = std::fs::read(r.root.join(".git/index")).unwrap(); + assert!( + r.snaps + .restore_reviewed( + &r.dir(), + &target.tree, + &recorded.tree, + &recorded.tree, + Arc::new(|| Ok(())) + ) + .await + .is_err() + ); + assert_eq!( + std::fs::read_to_string(r.root.join("a.txt")).unwrap(), + "replacement snapshot image" + ); + assert_eq!(std::fs::read(r.root.join(".git/index")).unwrap(), index); +} diff --git a/docs/rewind.md b/docs/rewind.md index 9db4a048f..9b783a685 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -326,3 +326,8 @@ consuming commit, so a database backup cannot replace a different native receipt An authorized later worktree removal retains referenced creation metadata for retry/recreation. Payloads, construction records and archived creation evidence remain protected until reference-aware cleanup is implemented. + +Git restore resolves file paths literally and verifies each blob image against its +Git object ID. Corrupt blob data or ambiguous image listings refuse before source +replacement. Retain the snapshot data for diagnosis; this check does not provide +the still-unfinished owned Git crash-recovery workflow. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 21b6913fb..f53268adb 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1649,3 +1649,42 @@ unchanged-main comparison or origin attribution. An initial integration compile used the async listing API incorrectly; it was corrected before the passing full worktree run. No critical/major review issue remains in this pre-effect native acknowledgement scope. The broader unfinished obligations above remain required. + + +## Exact Git image reads before owned-journal integration (2026-11-07) + +Git restore planning previously accepted the first loosely decoded tree-listing +record without checking the returned path or blob bytes against its object ID. +The shared `Shadow::blob` boundary now asks for a literal path and decodes exactly +one NUL-terminated record, preserving tabs/newlines in names. It validates safe +relative path components, supported modes and full object IDs. A valid directory +entry remains absence of a file image, preserving the existing planner distinction; +unsupported entry kinds and malformed/ambiguous records refuse. File, executable +and symbolic-link blob images are checked against Git's type/length-framed SHA-1 +or SHA-256 identity before being returned. Historical exclusion validation remains +first, and the user's index/refs/config are not changed. SHA-2 reuses the already +locked workspace 0.10.9 dependency; no dependency versions were upgraded. + +Unit coverage pins independent empty-blob identities for both formats and rejects +corrupt bytes, unsafe paths, mismatched names, multiple or unterminated records, +unsupported entries and invalid object IDs. Real Git restores cover literal glob/ +magic-looking names and tabs/newlines while preserving the user index. Another +real-repository test replaces a loose target blob with a different valid compressed +Git object; restore refuses and leaves both current source and user index unchanged. +The code-review skill was applied to all code, dependency/lock, test and spec changes, +checking exact byte parsing, no option/pathspec substitution, supported mode handling, +framed hashes and refusal before effects. This does not enable owned Git restore, +complete Git graph/native journal provenance or satisfy parent recovery acceptance. + +Verification: the full snapshot run passed 88 cases with one ignored fixture before +the additional real corrupt-object regression. The final 19-case Git integration +run passed with that regression included. Snapshot all-target Clippy passed with +warnings denied; formatting and diff checks passed. The initial locked command +required adding the existing SHA-2 dependency to the snapshot lock entry, which was +updated without changing versions before the passing locked run. Final strict +OpenSpec validation is recorded below. Full-workspace tests, real CLI/TUI/App +acceptance and native Linux/Windows execution were not run for this image-read +increment; owned Git recovery and its complete crash/platform acceptance remain open. + +Strict pinned OpenSpec passed all 117 items; spec lint reports 0 errors/25 warnings. +Generated SDK consistency passed with no API shape changes. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 43f98e58c..b42d7d08a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -448,3 +448,14 @@ unsettled. A cut after complete publication and before checkout settlement has positive exited-lifetime/resource evidence for the new metadata acknowledgement; partial acquisition and corrupt/incomplete records remain fenced until their separate exact recovery proof is implemented. + + +### Exact Git image materialization + +Git checkpoint images used for planning or owned-journal materialization must resolve +one exact literal path and retain its validated object ID and mode. NUL-delimited +listings must match that requested path, reject ambiguous/malformed records and +unsupported entry kinds, and never substitute a pathspec match. Blob contents must +match the Git SHA-1/SHA-256 object ID before becoming restore or recovery authority. +Existing historical exclusion evidence remains mandatory. Corrupt images fail +before source replacement; this does not enable owned Git journaling by itself. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index a12645f0f..dae6f7d25 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -130,6 +130,11 @@ - **AND** a retained acknowledgement stage SHALL grant no authority by itself and SHALL be preserved until separate acknowledged reference-aware reclamation - **AND** only explicit metadata reconciliation after committed receipt attestation SHALL resume interrupted acknowledgements without source replay +#### Scenario: Git journal images resolve exact immutable objects +- **WHEN** a Git checkpoint image is materialized for planning or recovery +- **THEN** the implementation SHALL resolve one exact literal path with a validated mode/object ID and historical exclusions +- **AND** malformed or ambiguous listings, unsupported entry kinds and blob contents that do not match the Git object ID SHALL refuse before source replacement + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index fa28cc196..35f2cb369 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -249,3 +249,12 @@ settlement. Actual native pre-publication kill/namespace replacement and real Gi removal/recreation cases cover this step. M2–M8 and parent tasks remain open for partial acquisition/corrupt construction, post-effect/owned Git integration, safe reclamation, complete clients and full crash/platform acceptance. + + +Git image boundary continuation: shadow-Git blob lookup now resolves an exact literal +NUL-delimited path, validates mode/object identity and checks the typed SHA-1/SHA-256 +blob hash. Malformed/ambiguous output, unsafe paths, unsupported entries and corrupt +blob bytes refuse before replacement. Real unusual-path and corrupted-loose-object +cases preserve the user's index and current source on failure. Historical exclusion +lookup remains mandatory. This is an image-materialization prerequisite for owned +Git journals, not their implementation; M2–M8 and all parent tasks remain open. From 7edb0ca6d5508c8feab2d8447ef0b6b1dc7c6f5a Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 02:08:13 -0400 Subject: [PATCH 20/87] Verify scoped Git tree catalogs before restore planning --- crates/cyber-snapshot/src/git.rs | 57 ++--- crates/cyber-snapshot/src/git/catalog.rs | 194 ++++++++++++++++++ crates/cyber-snapshot/src/git/images.rs | 125 ++++++----- crates/cyber-snapshot/src/git_history.rs | 20 +- crates/cyber-snapshot/tests/snapshots.rs | 135 ++++++++++++ .../implementation.md | 50 +++++ .../managed-recovery.md | 8 + .../specs/snapshots-checkpoints/spec.md | 2 + .../complete-daily-coding-surfaces/tasks.md | 10 + 9 files changed, 522 insertions(+), 79 deletions(-) create mode 100644 crates/cyber-snapshot/src/git/catalog.rs diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 8a7a279ef..54b7f9f69 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -2,9 +2,11 @@ use std::path::{Path, PathBuf}; use std::process::Stdio; +use std::sync::{Arc, Mutex}; use cyber_server::runtime::{FileDiff, Snapshot}; use sha1::{Digest, Sha1}; +mod catalog; #[path = "git_history.rs"] pub(crate) mod history; mod images; @@ -18,6 +20,7 @@ pub(crate) struct Shadow { common_dir: PathBuf, /// The Location relative to the worktree, as a pathspec (`.` for the root). pathspec: String, + catalogs: Arc>, } /// Environment variables that would redirect git away from the shadow repository. @@ -29,6 +32,10 @@ const GIT_ENV: &[&str] = &[ "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_NAMESPACE", + "GIT_LITERAL_PATHSPECS", + "GIT_GLOB_PATHSPECS", + "GIT_NOGLOB_PATHSPECS", + "GIT_ICASE_PATHSPECS", ]; /// Git for Windows does not accept verbatim (`\\?\`) paths as `GIT_DIR`, work tree or @@ -77,13 +84,16 @@ impl Shadow { let rel = location .strip_prefix(&worktree) .ok()? - .to_string_lossy() - .replace('\\', "/"); + .components() + .map(|part| part.as_os_str().to_str()) + .collect::>>()? + .join("/"); Some(Self { git_dir: git_path(&root.join(&project.id).join(hex)), worktree: git_path(&worktree), common_dir: git_path(&common_dir), pathspec: if rel.is_empty() { ".".into() } else { rel }, + catalogs: Arc::new(Mutex::new(catalog::CatalogCache::new())), }) } @@ -113,6 +123,7 @@ impl Shadow { .env("GIT_WORK_TREE", &self.worktree) .current_dir(&self.worktree) .args([ + "--literal-pathspecs", "-c", "core.autocrlf=false", "-c", @@ -177,6 +188,7 @@ impl Shadow { cmd.env_remove(name); } let out = cmd + .arg("--literal-pathspecs") .arg("-C") .arg(&self.worktree) .args([ @@ -219,25 +231,21 @@ impl Shadow { pub async fn changed(&self, from: &str, to: &str) -> Result, String> { let excluded = self.excluded_between(from, to).await?; - let out = self - .run(&[ - "diff", - "--name-only", - "--no-renames", - "-z", - from, - to, - "--", - &self.pathspec, - ]) - .await?; - Ok(split_z(&out) + let before = self.image_catalog(from).await?; + let after = self.image_catalog(to).await?; + let names: std::collections::BTreeSet<_> = + before.keys().chain(after.keys()).cloned().collect(); + Ok(names .into_iter() - .filter(|p| !history::excludes(&excluded, p)) + .filter(|path| { + before.get(path) != after.get(path) && !history::excludes(&excluded, path) + }) .collect()) } pub async fn diff(&self, from: &str, to: &str) -> Result, String> { + self.image_catalog(from).await?; + self.image_catalog(to).await?; let excluded = self.excluded_between(from, to).await?; let status = split_z( &self @@ -399,16 +407,17 @@ impl Shadow { /// `(mode, contents)` of a path in a tree, or `None` when absent. pub async fn blob(&self, tree: &str, path: &str) -> Result)>, String> { - self.recorded_exclusions(tree).await?; - let listing = self - .run(&["--literal-pathspecs", "ls-tree", "-z", tree, "--", path]) - .await?; - let Some((mode, object)) = images::entry(&listing, path)? else { + images::validate_path(path)?; + let catalog = self.image_catalog(tree).await?; + let Some((mode, object)) = catalog.get(path) else { return Ok(None); }; - let bytes = self.run(&["cat-file", "blob", &object]).await?; - images::verify(&object, &bytes)?; - Ok(Some((mode, bytes))) + if !matches!(mode.as_str(), "100644" | "100755" | "120000") { + return Err("Unsupported Git image entry".into()); + } + let bytes = self.run(&["cat-file", "blob", object]).await?; + images::verify(object, &bytes)?; + Ok(Some((mode.clone(), bytes))) } } diff --git a/crates/cyber-snapshot/src/git/catalog.rs b/crates/cyber-snapshot/src/git/catalog.rs new file mode 100644 index 000000000..499c74169 --- /dev/null +++ b/crates/cyber-snapshot/src/git/catalog.rs @@ -0,0 +1,194 @@ +//! Verified Git directory metadata. No source mutations or journal authority. +use super::{Shadow, history, images}; +use std::collections::BTreeMap; +use std::process::Stdio; +use std::sync::Arc; +pub(super) type CatalogCache = + BTreeMap<(std::path::PathBuf, std::path::PathBuf, String), Arc>; +use tokio::io::{AsyncRead, AsyncReadExt}; + +async fn bounded(reader: impl AsyncRead + Unpin, limit: usize) -> Result, String> { + let mut bytes = Vec::new(); + reader + .take(limit as u64 + 1) + .read_to_end(&mut bytes) + .await + .map_err(|error| error.to_string())?; + if bytes.len() > limit { + return Err("Git snapshot metadata output exceeds limit".into()); + } + Ok(bytes) +} + +impl Shadow { + pub(super) async fn run_bounded(&self, args: &[&str], limit: usize) -> Result, String> { + let mut child = self + .command(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + let stdout = child.stdout.take().ok_or("Missing Git metadata output")?; + let stderr = child + .stderr + .take() + .ok_or("Missing Git metadata diagnostics")?; + let reads = tokio::try_join!(bounded(stdout, limit), bounded(stderr, 64 * 1024)); + let (bytes, diagnostics) = match reads { + Ok(reads) => reads, + Err(error) => { + let _ = child.kill().await; + let _ = child.wait().await; + return Err(error); + } + }; + let status = child.wait().await.map_err(|error| error.to_string())?; + if !status.success() { + return Err(format!( + "Git snapshot metadata: {}", + String::from_utf8_lossy(&diagnostics).trim() + )); + } + Ok(bytes) + } + + pub(super) async fn image_catalog( + &self, + snapshot: &str, + ) -> Result, String> { + let key = ( + self.git_dir.clone(), + self.worktree.join(&self.pathspec), + snapshot.to_owned(), + ); + if let Some(catalog) = self + .catalogs + .lock() + .map_err(|_| "Git image catalog lock failed")? + .get(&key) + .cloned() + { + return Ok(catalog); + } + // Verifies immutable exclusion-tag bytes and the Location before resolving + // its tree. Bare legacy trees remain readable without invented exclusions. + let (root, excluded) = self.snapshot_metadata(snapshot).await?; + let excluded = excluded.unwrap_or_default(); + let catalog = Arc::new(self.read_catalog(root, &excluded).await?); + let mut cache = self + .catalogs + .lock() + .map_err(|_| "Git image catalog lock failed")?; + // Per-operation immutable inputs; eviction cannot invalidate held images. + if cache.len() >= 4 { + cache.clear(); + } + cache.insert(key, catalog.clone()); + Ok(catalog) + } + + async fn read_catalog( + &self, + root: String, + excluded: &[String], + ) -> Result { + let mut pending = vec![(String::new(), root, 0usize)]; + let mut catalog = BTreeMap::new(); + let mut size = 0usize; + let mut directories = 0usize; + let mut catalog_bytes = 0usize; + let mut pending_bytes = 0usize; + while let Some((prefix, object, depth)) = pending.pop() { + pending_bytes = pending_bytes.saturating_sub(prefix.len() + object.len()); + directories += 1; + if depth > 64 || directories > 65536 { + return Err("Git snapshot directory traversal exceeds limit".into()); + } + let bytes = self + .run_bounded(&["cat-file", "tree", &object], images::TREE_LIMIT) + .await?; + images::verify_object(&object, "tree", &bytes)?; + size = size + .checked_add(bytes.len()) + .ok_or("Git metadata size overflow")?; + if size > images::GRAPH_LIMIT { + return Err("Git snapshot metadata graph exceeds limit".into()); + } + for (name, mode, child) in images::tree_entries(&bytes, object.len() / 2)? { + let path = if prefix.is_empty() { + name + } else { + format!("{prefix}/{name}") + }; + images::validate_path(&path)?; + if history::excludes(excluded, &path) { + continue; + } + let within = self.pathspec == "." + || path == self.pathspec + || path.starts_with(&format!("{}/", self.pathspec)); + let ancestor = self.pathspec.starts_with(&format!("{path}/")); + if mode == "040000" { + if within || ancestor { + pending_bytes = pending_bytes + .checked_add(path.len() + child.len()) + .ok_or("Git metadata size overflow")?; + if pending_bytes > images::GRAPH_LIMIT { + return Err("Git directory work queue exceeds limit".into()); + } + pending.push((path, child, depth + 1)); + } + } else if within { + catalog_bytes = catalog_bytes + .checked_add(path.len() + mode.len() + child.len()) + .ok_or("Git image catalog size overflow")?; + if catalog_bytes > images::GRAPH_LIMIT { + return Err("Git image catalog byte limit exceeded".into()); + } + if catalog.len() >= 65536 || catalog.insert(path, (mode, child)).is_some() { + return Err( + "Git image catalog exceeds limits or contains duplicates".into() + ); + } + } + } + } + Ok(catalog) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn over_limit_git_output_returns_failure_and_the_reader_remains_usable() { + let temp = tempfile::tempdir().unwrap(); + let shadow = Shadow { + git_dir: temp.path().join("shadow"), + worktree: temp.path().to_path_buf(), + common_dir: temp.path().join("common"), + pathspec: ".".into(), + catalogs: Default::default(), + }; + assert!( + shadow + .run_bounded(&["--version"], 0) + .await + .unwrap_err() + .contains("exceeds limit") + ); + assert!( + shadow + .run_bounded(&["--version"], 1024) + .await + .unwrap() + .starts_with(b"git version ") + ); + } + + #[tokio::test] + async fn metadata_reads_reject_overflow_without_accepting_a_prefix() { + assert_eq!(bounded(&b"abc"[..], 3).await.unwrap(), b"abc"); + assert!(bounded(&b"abcd"[..], 3).await.is_err()); + } +} diff --git a/crates/cyber-snapshot/src/git/images.rs b/crates/cyber-snapshot/src/git/images.rs index 038c8f462..8145b8a7b 100644 --- a/crates/cyber-snapshot/src/git/images.rs +++ b/crates/cyber-snapshot/src/git/images.rs @@ -2,57 +2,78 @@ use sha1::{Digest, Sha1}; use sha2::Sha256; -pub(super) fn entry(listing: &[u8], path: &str) -> Result, String> { +pub(super) type Catalog = std::collections::BTreeMap; +pub(super) const TREE_LIMIT: usize = 8 * 1024 * 1024; +pub(super) const GRAPH_LIMIT: usize = 64 * 1024 * 1024; + +pub(super) fn validate_path(path: &str) -> Result<(), String> { if path.is_empty() || path.len() > 4096 || path.contains('\0') + || path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") || std::path::Path::new(path) .components() .any(|component| !matches!(component, std::path::Component::Normal(_))) { return Err("Invalid Git image path".into()); } - if listing.is_empty() { - return Ok(None); - } - let record = listing - .strip_suffix(&[0]) - .ok_or("Unterminated Git image listing")?; - if record.contains(&0) { - return Err("Ambiguous Git image listing".into()); - } - let split = record - .iter() - .position(|byte| *byte == b'\t') - .ok_or("Malformed Git image listing")?; - if &record[split + 1..] != path.as_bytes() { - return Err("Git image path differs from the requested literal path".into()); + Ok(()) +} +/// Parse raw Git tree records without treating binary object IDs as delimiters. +pub(super) fn tree_entries( + mut bytes: &[u8], + width: usize, +) -> Result, String> { + if !matches!(width, 20 | 32) { + return Err("Invalid Git tree object width".into()); } - let metadata = - std::str::from_utf8(&record[..split]).map_err(|_| "Invalid Git image metadata")?; - let fields: Vec<_> = metadata.split(' ').collect(); - if let ["040000", "tree", object] = fields.as_slice() { - if !oid(object) { - return Err("Invalid Git directory image identity".into()); + let mut entries = Vec::new(); + let mut names = std::collections::BTreeSet::new(); + while !bytes.is_empty() { + let space = bytes + .iter() + .position(|byte| *byte == b' ') + .ok_or("Malformed Git tree mode")?; + let mode = std::str::from_utf8(&bytes[..space]).map_err(|_| "Invalid Git tree mode")?; + let tail = &bytes[space + 1..]; + let nul = tail + .iter() + .position(|byte| *byte == 0) + .ok_or("Unterminated Git tree path")?; + let name = std::str::from_utf8(&tail[..nul]) + .map_err(|_| "Non-UTF-8 Git tree path is unsupported")?; + validate_path(name)?; + if name.contains('/') || !names.insert(name.to_owned()) { + return Err("Ambiguous Git tree path".into()); } - return Ok(None); // A directory is not a file image at this exact path. - } - let [mode, "blob", object] = fields.as_slice() else { - return Err("Unsupported Git image entry".into()); - }; - if !matches!(*mode, "100644" | "100755" | "120000") || !oid(object) { - return Err("Invalid Git image mode or object identity".into()); + let object = tail + .get(nul + 1..nul + 1 + width) + .ok_or("Truncated Git tree object")?; + let mode = match mode { + "40000" | "040000" => "040000", + "100644" | "100755" | "120000" | "160000" => mode, + _ => return Err("Unsupported Git tree mode".into()), + }; + entries.push(( + name.into(), + mode.into(), + object.iter().map(|byte| format!("{byte:02x}")).collect(), + )); + if entries.len() > 65536 { + return Err("Git tree entry limit exceeded".into()); + } + bytes = &tail[nul + 1 + width..]; } - Ok(Some(((*mode).into(), (*object).into()))) -} -fn oid(object: &str) -> bool { - matches!(object.len(), 40 | 64) - && object - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + Ok(entries) } + pub(super) fn verify(object: &str, bytes: &[u8]) -> Result<(), String> { - let header = format!("blob {}\0", bytes.len()); + verify_object(object, "blob", bytes) +} +pub(super) fn verify_object(object: &str, kind: &str, bytes: &[u8]) -> Result<(), String> { + let header = format!("{kind} {}\0", bytes.len()); let actual = match object.len() { 40 => { let mut hash = Sha1::new(); @@ -78,25 +99,25 @@ pub(super) fn verify(object: &str, bytes: &[u8]) -> Result<(), String> { mod tests { use super::*; #[test] - fn exact_nul_records_preserve_unusual_paths_and_refuse_ambiguous_or_foreign_images() { - let object = "a".repeat(40); - let path = ":(glob)*\tline\nname"; - let record = format!("100755 blob {object}\t{path}\0"); + fn binary_tree_records_preserve_literal_names_and_bound_object_width() { + let name = ":(glob)*\tline\nname"; + let mut record = format!("100755 {name}\0").into_bytes(); + record.extend([0x11; 20]); assert_eq!( - entry(record.as_bytes(), path).unwrap(), - Some(("100755".into(), object.clone())) + tree_entries(&record, 20).unwrap(), + [(name.into(), "100755".into(), "11".repeat(20))] ); - assert_eq!(entry(b"", path).unwrap(), None); - assert!(entry(b"", "../outside").is_err()); + assert!(tree_entries(&record, 32).is_err()); + assert!(tree_entries(&[record.clone(), record.clone()].concat(), 20).is_err()); + assert!(tree_entries(b"40000 ../outside\0", 20).is_err()); + assert!(validate_path("../outside").is_err()); + assert!(validate_path("a//b").is_err()); + let mut wide = b"40000 nested\0".to_vec(); + wide.extend([0; 32]); assert_eq!( - entry(format!("040000 tree {object}\t{path}\0").as_bytes(), path).unwrap(), - None + tree_entries(&wide, 32).unwrap(), + [("nested".into(), "040000".into(), "00".repeat(32))] ); - assert!(entry(record.as_bytes(), "different").is_err()); - assert!(entry(record.trim_end_matches('\0').as_bytes(), path).is_err()); - assert!(entry(format!("{record}{record}").as_bytes(), path).is_err()); - assert!(entry(format!("160000 commit {object}\t{path}\0").as_bytes(), path).is_err()); - assert!(entry(format!("100644 blob invalid\t{path}\0").as_bytes(), path).is_err()); } #[test] fn git_object_hashes_are_checked_with_the_type_and_length_frame() { diff --git a/crates/cyber-snapshot/src/git_history.rs b/crates/cyber-snapshot/src/git_history.rs index 8c1951f8a..537e1a79e 100644 --- a/crates/cyber-snapshot/src/git_history.rs +++ b/crates/cyber-snapshot/src/git_history.rs @@ -129,12 +129,23 @@ impl Shadow { &self, id: &str, ) -> Result>, String> { + self.snapshot_metadata(id) + .await + .map(|(_, excluded)| excluded) + } + + /// Return the root identity from the same verified bytes as the exclusions. + /// Resolving the tag again would allow a later substitution to change its root. + pub(super) async fn snapshot_metadata( + &self, + id: &str, + ) -> Result<(String, Option>), String> { if !oid(id) { return Err("Invalid snapshot identity".into()); } let kind = self.run(&["cat-file", "-t", id]).await?; if kind == b"tree\n" { - return Ok(None); + return Ok((id.to_owned(), None)); } if kind != b"tag\n" { return Err("Snapshot identity is not a tree or exclusion record".into()); @@ -148,7 +159,10 @@ impl Shadow { if size > LIMIT + 512 { return Err("Snapshot exclusion record exceeds limit".into()); } - let bytes = self.run(&["cat-file", "tag", id]).await?; + let bytes = self + .run_bounded(&["cat-file", "tag", id], LIMIT + 512) + .await?; + super::images::verify_object(id, "tag", &bytes)?; let text = std::str::from_utf8(&bytes).map_err(|e| e.to_string())?; let (header, body) = text .split_once("\n\n") @@ -168,7 +182,7 @@ impl Shadow { "Snapshot exclusion record belongs to another Location or is corrupt".into(), ); } - Ok(Some(record.excluded)) + Ok((record.tree, Some(record.excluded))) } pub(super) async fn excluded_between( &self, diff --git a/crates/cyber-snapshot/tests/snapshots.rs b/crates/cyber-snapshot/tests/snapshots.rs index 1fbd441c2..cccc8dbf7 100644 --- a/crates/cyber-snapshot/tests/snapshots.rs +++ b/crates/cyber-snapshot/tests/snapshots.rs @@ -32,6 +32,27 @@ fn git(dir: &Path, args: &[&str]) -> String { } impl Repo { + fn shadow_directory(&self) -> PathBuf { + std::fs::read_dir(self.data.join("snapshot")) + .unwrap() + .flat_map(|entry| std::fs::read_dir(entry.unwrap().path()).unwrap()) + .map(|entry| entry.unwrap().path()) + .find(|path| path.join("HEAD").exists()) + .unwrap() + } + fn shadow_object(&self, revision: &str) -> PathBuf { + let shadow = self.shadow_directory(); + let output = Command::new("git") + .arg("--git-dir") + .arg(&shadow) + .args(["rev-parse", revision]) + .output() + .unwrap(); + assert!(output.status.success()); + let id = String::from_utf8(output.stdout).unwrap(); + let id = id.trim(); + shadow.join("objects").join(&id[..2]).join(&id[2..]) + } fn new() -> Self { let tmp = tempfile::tempdir().unwrap(); let base = std::fs::canonicalize(tmp.path()).unwrap(); @@ -581,3 +602,117 @@ async fn corrupt_git_snapshot_blob_refuses_restore_and_preserves_current_source( ); assert_eq!(std::fs::read(r.root.join(".git/index")).unwrap(), index); } + +#[cfg(unix)] +#[tokio::test] +async fn corrupt_git_tag_root_and_nested_tree_cannot_turn_restore_into_a_silent_noop() { + use std::os::unix::fs::PermissionsExt; + for suffix in ["", "^{tree}", ":nested"] { + let r = Repo::new(); + std::fs::create_dir(r.root.join("nested")).unwrap(); + std::fs::write(r.root.join("nested/file"), "before graph image").unwrap(); + let target = r.snaps.track(&r.dir()).await.unwrap().unwrap(); + std::fs::write(r.root.join("nested/file"), "after graph image").unwrap(); + let recorded = r.snaps.track(&r.dir()).await.unwrap().unwrap(); + let original = r.shadow_object(&format!("{}{suffix}", target.tree)); + let replacement = + std::fs::read(r.shadow_object(&format!("{}{suffix}", recorded.tree))).unwrap(); + std::fs::set_permissions(&original, std::fs::Permissions::from_mode(0o600)).unwrap(); + std::fs::write(original, replacement).unwrap(); + let index = std::fs::read(r.root.join(".git/index")).unwrap(); + assert!( + r.snaps + .restore_reviewed( + &r.dir(), + &target.tree, + &recorded.tree, + &recorded.tree, + Arc::new(|| Ok(())) + ) + .await + .is_err(), + "{suffix}" + ); + assert_eq!( + std::fs::read_to_string(r.root.join("nested/file")).unwrap(), + "after graph image" + ); + assert_eq!(std::fs::read(r.root.join(".git/index")).unwrap(), index); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn git_location_paths_are_literal_and_preserve_backslashes_without_leaking_sibling_files() { + let r = Repo::new(); + for scope in ["[ab]", "back\\slash", "a", "b"] { + std::fs::create_dir(r.root.join(scope)).unwrap(); + std::fs::write(r.root.join(scope).join("file"), "before scoped image").unwrap(); + } + let index = std::fs::read(r.root.join(".git/index")).unwrap(); + for scope in ["[ab]", "back\\slash"] { + let location = r.root.join(scope).to_str().unwrap().to_owned(); + let target = r.snaps.track(&location).await.unwrap().unwrap(); + std::fs::write(r.root.join(scope).join("file"), "after scoped image").unwrap(); + std::fs::write(r.root.join("a/file"), "later sibling owner").unwrap(); + std::fs::write(r.root.join("b/file"), "later sibling owner").unwrap(); + let recorded = r.snaps.track(&location).await.unwrap().unwrap(); + assert_eq!( + r.snaps + .changed(&location, &target.tree, &recorded.tree) + .await + .unwrap(), + [format!("{scope}/file")] + ); + r.snaps + .restore_reviewed( + &location, + &target.tree, + &recorded.tree, + &recorded.tree, + Arc::new(|| Ok(())), + ) + .await + .unwrap(); + assert_eq!( + std::fs::read_to_string(r.root.join(scope).join("file")).unwrap(), + "before scoped image" + ); + assert_eq!( + std::fs::read_to_string(r.root.join("a/file")).unwrap(), + "later sibling owner" + ); + assert_eq!( + std::fs::read_to_string(r.root.join("b/file")).unwrap(), + "later sibling owner" + ); + } + assert_eq!(std::fs::read(r.root.join(".git/index")).unwrap(), index); +} + +#[cfg(unix)] +#[test] +#[ignore = "subprocess fixture invoked by pathspec environment test"] +fn snapshot_pathspec_environment_child() { + git_location_paths_are_literal_and_preserve_backslashes_without_leaking_sibling_files(); +} + +#[cfg(unix)] +#[test] +fn snapshot_capture_does_not_inherit_glob_pathspec_environment() { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "snapshot_pathspec_environment_child", + "--ignored", + ]) + .env("GIT_GLOB_PATHSPECS", "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index f53268adb..f4124cda8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1688,3 +1688,53 @@ increment; owned Git recovery and its complete crash/platform acceptance remain Strict pinned OpenSpec passed all 117 items; spec lint reports 0 errors/25 warnings. Generated SDK consistency passed with no API shape changes. + + +## Verified scoped Git catalogs (2026-11-07) + +The next Git integration prerequisite replaces changed-path derivation from Git +diff output with an immutable literal-path catalog. Exclusion-tag payloads and +every eligible scoped directory tree are checked against type/length-framed Git +object identities before paths, modes and leaf IDs are used. Raw binary tree +records preserve unusual names and reject malformed modes, unsafe paths, duplicate +names and truncated IDs. Leaf bytes remain verified when materialized. This does +not assert that all unchanged blobs were read or enable the native Git journal. + +Each tree read is bounded to 8 MiB; traversed graph, catalog contents and pending +path/object queue each have a 64 MiB byte bound. Traversal limits depth to 64 and +directories/catalog entries to 65,536. At most four catalogs are cached per Shadow +instance, keyed by shadow repository, Location and snapshot identity. Excess +subprocess output is rejected after killing and awaiting the child, with bounded +diagnostics. These are metadata bounds, not source effect or reclamation authority. +CLI path selectors are literal and inherited pathspec flags are removed. Location +components retain literal Unix backslashes rather than changing them to separators. + +Regression coverage uses real Git repositories with different valid compressed +objects substituted for exclusion tags, root trees and nested trees. Each restore +refuses while preserving source and the user index. Literal glob-shaped scopes and +backslash names restore only their exact Location and preserve sibling owner edits. +A separate subprocess repeats this under inherited GIT_GLOB_PATHSPECS. Unit tests +cover binary SHA-1/SHA-256 tree IDs, duplicate/truncated records and metadata +overflow, including an actual Git child followed by a successful read. + +The code-review skill was applied to all changed source, tests and specs. Review +checked typed object hashes, scoped traversal, byte/entry bounds, exact paths, +subprocess failure cleanup and absence of new native acknowledgement authority. +No critical or major finding remains in this catalog increment. The new catalog +module remains inside the existing Git adapter; runtime ports and schemas are +unchanged. Owned Git source/link handling and durable original Git context in native +journals remain the next integration work; all parent recovery tasks stay open. + +Verification on macOS arm64: locked snapshot tests passed 94 cases with two ignored +fixtures (the pathspec subprocess fixture is invoked by its parent). All-target +snapshot Clippy passed with warnings denied. An initial collapsible-if Clippy +finding was resolved before the passing final run. Full-workspace tests, SDK +execution tests, real CLI/TUI/App recovery and Linux/Windows execution were not run +for this increment; no broader acceptance is inferred. + +Final review found and removed a second tag resolution between verification and +root traversal. Root identity and exclusions now come from the same verified tag +bytes; legacy bare trees retain their exact supplied identity. This closes the +substitution window rather than trusting a subsequent rev-parse result. Final +formatting, generated SDK consistency and diff checks passed. Strict pinned +OpenSpec passed 117 items; spec lint reports 0 errors/25 warnings. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index b42d7d08a..5cdec9681 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -459,3 +459,11 @@ unsupported entry kinds, and never substitute a pathspec match. Blob contents mu match the Git SHA-1/SHA-256 object ID before becoming restore or recovery authority. Existing historical exclusion evidence remains mandatory. Corrupt images fail before source replacement; this does not enable owned Git journaling by itself. + +Git image preparation also verifies exclusion-tag identities and the complete eligible +scoped directory-tree graph before deriving changed paths. A verified in-memory +catalog binds literal paths, modes and leaf object IDs to that graph, so a corrupt +nested tree cannot make a restore appear to have no changes. Metadata reads are +bounded and over-limit subprocess output is killed and awaited. This catalog is +preparation input for the future Git journal bridge, not native origin authority; +source mutation must still run under the retained synchronous filesystem worker. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index dae6f7d25..e2e37d5ed 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -134,6 +134,8 @@ - **WHEN** a Git checkpoint image is materialized for planning or recovery - **THEN** the implementation SHALL resolve one exact literal path with a validated mode/object ID and historical exclusions - **AND** malformed or ambiguous listings, unsupported entry kinds and blob contents that do not match the Git object ID SHALL refuse before source replacement +- **AND** exclusion tags and every scoped directory tree SHALL match their object IDs before changed paths are derived from a verified literal-path catalog +- **AND** metadata traversal and subprocess output SHALL be bounded, with over-limit children killed and awaited #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 35f2cb369..c86f0d93a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -258,3 +258,13 @@ blob bytes refuse before replacement. Real unusual-path and corrupted-loose-obje cases preserve the user's index and current source on failure. Historical exclusion lookup remains mandatory. This is an image-materialization prerequisite for owned Git journals, not their implementation; M2–M8 and all parent tasks remain open. + + +Verified Git catalog continuation: changed-path derivation now uses a bounded +Location-scoped catalog whose exclusion tag and every traversed directory tree +match their typed Git object IDs. Metadata subprocess overflow refuses after +killing/awaiting the child. Literal Location selectors preserve Unix backslashes +and glob-shaped names independently of inherited pathspec environment. Real +corrupt tag/root/nested-tree restores refuse without changing source or user index; +scoped restores preserve sibling files. This remains preparation for owned Git +journals; M2–M8 and parent tasks are unchanged and incomplete. From d37c0ce663e66e8e25b6f57fa5095c67c732717c Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 02:19:08 -0400 Subject: [PATCH 21/87] Import verified Git file images through native recovery journals --- crates/cyber-snapshot/src/copy/git.rs | 220 +++++++++ crates/cyber-snapshot/src/copy/git/tests.rs | 448 ++++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 1 + .../src/copy/resolution_tests.rs | 2 + crates/cyber-snapshot/src/copy/restore.rs | 35 +- crates/cyber-snapshot/src/git.rs | 2 + crates/cyber-snapshot/src/git/prepared.rs | 134 ++++++ crates/cyber-snapshot/src/lib.rs | 46 +- .../implementation.md | 61 +++ .../managed-recovery.md | 14 + .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 12 + 12 files changed, 959 insertions(+), 23 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/git.rs create mode 100644 crates/cyber-snapshot/src/copy/git/tests.rs create mode 100644 crates/cyber-snapshot/src/git/prepared.rs diff --git a/crates/cyber-snapshot/src/copy/git.rs b/crates/cyber-snapshot/src/copy/git.rs new file mode 100644 index 000000000..0b3c48799 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/git.rs @@ -0,0 +1,220 @@ +//! Import verified Git images into the retained native journal, with original IDs. +use super::restore::{Change, RestoreContext}; +use super::storage::{self, Result}; +use super::{ENTRY_LIMIT, Entry, Store}; +use crate::git::prepared::{Image, PreparedRestore}; +use cyber_server::runtime::{RestoreError, SnapshotWork}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::os::unix::fs::MetadataExt; +use std::path::{Component, Path, PathBuf}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct GitContext { + pub worktree: PathBuf, + pub prefix: String, +} + +impl RestoreContext { + pub(super) fn validate(&self, location: &Path, version: u32) -> Result<()> { + let ids = [&self.target, &self.recorded, &self.expected]; + if let Some(git) = &self.git { + if version != 4 + || !git.worktree.is_absolute() + || git + .worktree + .components() + .any(|part| matches!(part, Component::ParentDir | Component::CurDir)) + || git.worktree.join(&git.prefix) != location + || !ids.iter().all(|id| git_id(id) && id.len() == ids[0].len()) + { + return Err("Invalid Git recovery snapshot context".into()); + } + if !git.prefix.is_empty() { + super::tree::safe_path(&git.prefix)?; + } + } else if !ids + .iter() + .all(|id| id.strip_prefix("copy:").is_some_and(storage::valid_digest)) + { + return Err("Invalid recovery snapshot context".into()); + } + if self.exclusions.len() > ENTRY_LIMIT * 3 { + return Err("Recovery exclusion inventory exceeds limit".into()); + } + Ok(()) + } +} + +fn git_id(id: &str) -> bool { + matches!(id.len(), 40 | 64) + && id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} +fn matches(entry: Option<&Entry>, image: Option<&Image>) -> bool { + match (entry, image) { + (None, None) => true, + (Some(entry), Some(image)) => { + entry.length == image.bytes.len() as u64 + && entry.object == storage::digest(&image.bytes) + && (entry.mode & 0o111 != 0) == (image.mode == "100755") + } + _ => false, + } +} + +impl Store { + /// Called only inside a retained synchronous worker. Ordinary Git admission + /// stays disabled until link import and full expected-state scanning are ready. + pub(crate) fn restore_git_worker( + &self, + prepared: PreparedRestore, + worker: Option<&mut SnapshotWork<'_>>, + ) -> std::result::Result, RestoreError> { + if let Some(worker) = worker.as_ref() { + worker.authorize().map_err(RestoreError::Failed)?; + } + let (changes, context) = self.plan_git(&prepared)?; + if changes.is_empty() { + return Ok(vec![]); + } + let id = format!("restore_{}", ulid::Ulid::new()); + let mut journal = self + .prepare_journal(changes, &id, worker.as_deref(), Some(context), None) + .map_err(RestoreError::Failed)?; + self.execute_journal(&id, &mut journal, &|| Ok(()), worker) + } + + fn plan_git( + &self, + prepared: &PreparedRestore, + ) -> std::result::Result<(Vec, RestoreContext), RestoreError> { + let context = RestoreContext { + target: prepared.target.id.clone(), + recorded: prepared.recorded.id.clone(), + expected: prepared.expected.id.clone(), + exclusions: prepared.exclusions.clone(), + git: Some(GitContext { + worktree: prepared.worktree.clone(), + prefix: prepared.prefix.clone(), + }), + }; + context + .validate(&self.root, 4) + .map_err(RestoreError::Failed)?; + if prepared.directory != self.root { + return Err(RestoreError::Failed( + "Git handoff belongs to another native Location".into(), + )); + } + self.ensure_recovered().map_err(RestoreError::Failed)?; + // Known image validation precedes any source effect. Link support is required + // before enabling the runtime gate; this import currently handles files. + for tree in [&prepared.target, &prepared.recorded, &prepared.expected] { + for (path, image) in &tree.images { + image.verify().map_err(RestoreError::Failed)?; + self.check_source_path(path).map_err(RestoreError::Failed)?; + if !matches!(image.mode.as_str(), "100644" | "100755") { + return Err(RestoreError::Failed( + "Native Git link import is not yet available".into(), + )); + } + if image.bytes.len() as u64 > self.options.max_file { + return Err(RestoreError::Failed( + "Git journal image exceeds file limit".into(), + )); + } + } + } + let paths: BTreeSet<_> = prepared + .target + .images + .keys() + .chain(prepared.recorded.images.keys()) + .chain(prepared.expected.images.keys()) + .cloned() + .collect(); + if paths.len() > ENTRY_LIMIT { + return Err(RestoreError::Failed( + "Git journal path count exceeds limit".into(), + )); + } + let mut changes = Vec::new(); + let mut conflicts = Vec::new(); + for path in paths { + let current = self.current(&path).map_err(RestoreError::Failed)?; + if !matches(current.as_ref(), prepared.expected.images.get(&path)) { + return Err(RestoreError::Stale); + } + let base = prepared.recorded.images.get(&path); + let want = prepared.target.images.get(&path); + if base == want + || prepared + .allowed + .as_ref() + .is_some_and(|paths| !paths.contains(&path)) + || matches(current.as_ref(), want) + { + continue; + } + let next = if matches(current.as_ref(), base) { + want.map(|image| self.git_entry(image, current.as_ref())) + .transpose() + .map_err(RestoreError::Failed)? + } else { + let base = base + .map(|image| self.git_entry(image, current.as_ref())) + .transpose() + .map_err(RestoreError::Failed)?; + let want = want + .map(|image| self.git_entry(image, current.as_ref())) + .transpose() + .map_err(RestoreError::Failed)?; + match self + .merged(current.as_ref(), base.as_ref(), want.as_ref()) + .map_err(RestoreError::Failed)? + { + Some(entry) => Some(entry), + None => { + conflicts.push(path); + continue; + } + } + }; + changes.push(Change { + path, + before: current, + after: next, + }); + } + if !conflicts.is_empty() { + return Err(RestoreError::Conflict(conflicts)); + } + Ok((changes, context)) + } + + fn git_entry(&self, image: &Image, current: Option<&Entry>) -> Result { + let mut entry = self.object(&image.bytes)?; + let owner = self + .work + .try_clone() + .map_err(storage::fail)? + .into_std_file() + .metadata() + .map_err(storage::fail)?; + entry.mode = current.map_or(0o644, |entry| entry.mode); + entry.mode = if image.mode == "100755" { + entry.mode | 0o111 + } else { + entry.mode & !0o111 + }; + entry.uid = current.map_or(owner.uid(), |entry| entry.uid); + entry.gid = current.map_or(owner.gid(), |entry| entry.gid); + Ok(entry) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs new file mode 100644 index 000000000..9e5dc45db --- /dev/null +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -0,0 +1,448 @@ +//! Real Git image preparation followed by synchronous native journal execution. +use super::*; +use cyber_server::runtime::{SnapshotRestoreRequest, Snapshots}; +use std::os::unix::fs::PermissionsExt; +use std::sync::Arc; + +struct Repo { + _temp: tempfile::TempDir, + root: PathBuf, + data: PathBuf, + snapshots: Arc, +} +impl Repo { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("work"); + let data = base.join("data"); + std::fs::create_dir(&root).unwrap(); + assert!( + std::process::Command::new("git") + .args(["init", "--quiet"]) + .arg(&root) + .status() + .unwrap() + .success() + ); + let snapshots = crate::GitSnapshots::new(data.clone(), Arc::new(|_| serde_json::json!({}))); + Self { + _temp: temp, + root, + data, + snapshots, + } + } + async fn track(&self, directory: &Path) -> String { + self.snapshots + .track(directory.to_str().unwrap()) + .await + .unwrap() + .unwrap() + .tree + } + fn store(&self, directory: &Path) -> Store { + Store::open( + &self.data, + directory.to_str().unwrap(), + super::super::Options { + max_file: 65536, + max_total: 8 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap() + } + async fn prepare( + &self, + directory: &Path, + target: &str, + recorded: &str, + expected: &str, + ) -> PreparedRestore { + self.snapshots + .shadow(directory.to_str().unwrap()) + .unwrap() + .prepare_restore( + &SnapshotRestoreRequest { + directory: directory.to_str().unwrap().into(), + target: target.into(), + recorded: recorded.into(), + expected: expected.into(), + paths: None, + }, + 65536, + 8 * 1024 * 1024, + ) + .await + .unwrap() + } +} + +#[tokio::test] +async fn native_git_import_restores_files_modes_creations_deletions_and_retains_original_ids() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + std::fs::set_permissions( + repo.root.join("file"), + std::fs::Permissions::from_mode(0o755), + ) + .unwrap(); + std::fs::create_dir(repo.root.join("deleted")).unwrap(); + std::fs::write(repo.root.join("deleted/file"), b"restore parent").unwrap(); + assert!( + std::process::Command::new("git") + .arg("-C") + .arg(&repo.root) + .args(["add", "file"]) + .status() + .unwrap() + .success() + ); + let user_index = std::fs::read(repo.root.join(".git/index")).unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + std::fs::set_permissions( + repo.root.join("file"), + std::fs::Permissions::from_mode(0o644), + ) + .unwrap(); + std::fs::remove_file(repo.root.join("deleted/file")).unwrap(); + std::fs::remove_dir(repo.root.join("deleted")).unwrap(); + std::fs::write(repo.root.join("added"), b"added").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + // Native execution must not ask Git for object bytes or use its index again. + std::fs::rename(&shadow.git_dir, shadow.git_dir.with_extension("retained")).unwrap(); + let store = repo.store(&repo.root); + assert_eq!( + store.restore_git_worker(prepared, None).unwrap(), + ["added", "deleted/file", "file"] + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"before"); + assert_ne!( + std::fs::metadata(repo.root.join("file")) + .unwrap() + .permissions() + .mode() + & 0o111, + 0 + ); + assert_eq!( + std::fs::read(repo.root.join("deleted/file")).unwrap(), + b"restore parent" + ); + assert!(!repo.root.join("added").exists()); + assert_eq!( + std::fs::read(repo.root.join(".git/index")).unwrap(), + user_index + ); + let journals = store.recovery().unwrap(); + assert_eq!(journals.len(), 1); + let (dir, journal) = store.journal(&journals[0].id).unwrap(); + assert_eq!(journal.version, 4); + let context = journal.context.as_ref().unwrap(); + assert_eq!( + (&context.target, &context.recorded, &context.expected), + (&target, &recorded, &recorded) + ); + assert_eq!(context.git.as_ref().unwrap().worktree, repo.root); + assert!(journal.completed); + assert!(journal.ownership.is_some()); + assert_eq!(storage::read(&dir, "stage_2", 65536).unwrap(), b"after"); + assert!( + !repo + .snapshots + .owned_restore_available(repo.root.to_str().unwrap()) + ); + let mut downgraded = journal.clone(); + downgraded.version = 3; + Store::record(&dir, &downgraded).unwrap(); + assert!(store.journal(&journals[0].id).is_err()); + let mut missing_context = journal.clone(); + missing_context.context = None; + Store::record(&dir, &missing_context).unwrap(); + assert!(store.journal(&journals[0].id).is_err()); + let mut missing_ownership = journal; + missing_ownership.ownership = None; + Store::record(&dir, &missing_ownership).unwrap(); + assert!(store.journal(&journals[0].id).is_err()); +} + +#[tokio::test] +async fn native_git_import_merges_owner_edits_and_rejects_stale_handoffs() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"one\ntwo\nthree\n").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"ONE\ntwo\nthree\n").unwrap(); + let recorded = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"ONE\ntwo\nTHREE\n").unwrap(); + let expected = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &expected) + .await; + let store = repo.store(&repo.root); + store.restore_git_worker(prepared, None).unwrap(); + assert_eq!( + std::fs::read(repo.root.join("file")).unwrap(), + b"one\ntwo\nTHREE\n" + ); + drop(store); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &expected) + .await; + std::fs::write(repo.root.join("file"), b"later owner").unwrap(); + let store = repo.store(&repo.root); + let before = store.recovery().unwrap().len(); + assert!(matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + )); + assert_eq!(store.recovery().unwrap().len(), before); + assert_eq!( + std::fs::read(repo.root.join("file")).unwrap(), + b"later owner" + ); +} + +#[tokio::test] +async fn native_git_import_keeps_subdirectory_and_historical_exclusion_boundaries() { + let repo = Repo::new(); + let directory = repo.root.join("[ab]"); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("file"), b"before").unwrap(); + std::fs::write(directory.join("ignored"), b"owner ignored before").unwrap(); + std::fs::write(directory.join(".gitignore"), b"ignored\n").unwrap(); + let target = repo.track(&directory).await; + std::fs::write(directory.join("file"), b"after").unwrap(); + std::fs::write(directory.join("ignored"), b"later ignored owner").unwrap(); + std::fs::write(repo.root.join("sibling"), b"sibling owner").unwrap(); + let recorded = repo.track(&directory).await; + let prepared = repo + .prepare(&directory, &target, &recorded, &recorded) + .await; + let store = repo.store(&directory); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); + assert_eq!(std::fs::read(directory.join("file")).unwrap(), b"before"); + assert_eq!( + std::fs::read(directory.join("ignored")).unwrap(), + b"later ignored owner" + ); + assert_eq!( + std::fs::read(repo.root.join("sibling")).unwrap(), + b"sibling owner" + ); + let recovery = store.recovery().unwrap(); + let (_, journal) = store.journal(&recovery[0].id).unwrap(); + let context = journal.context.unwrap(); + assert_eq!(context.git.as_ref().unwrap().prefix, "[ab]"); + assert!( + context + .exclusions + .iter() + .any(|path| path.contains("ignored")) + ); + let mut invalid = context.clone(); + invalid.git.as_mut().unwrap().prefix = "../foreign".into(); + assert!(invalid.validate(&directory, 4).is_err()); + invalid = context; + invalid.target = format!("copy:{}", "0".repeat(64)); + assert!(invalid.validate(&directory, 4).is_err()); +} + +#[tokio::test] +async fn native_git_import_refuses_link_images_before_any_journal_or_source_effect() { + let repo = Repo::new(); + std::os::unix::fs::symlink("destination", repo.root.join("link")).unwrap(); + let target = repo.track(&repo.root).await; + std::fs::remove_file(repo.root.join("link")).unwrap(); + std::fs::write(repo.root.join("link"), b"current owner").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert!( + matches!(store.restore_git_worker(prepared, None), Err(RestoreError::Failed(error)) if error.contains("link import")) + ); + assert!(store.recovery().unwrap().is_empty()); + assert_eq!( + std::fs::read(repo.root.join("link")).unwrap(), + b"current owner" + ); +} + +#[test] +fn legacy_copy_context_keeps_its_encoding_and_cannot_be_reinterpreted_as_git() { + let value = serde_json::json!({"target": format!("copy:{}", "a".repeat(64)), + "recorded": format!("copy:{}", "b".repeat(64)), "expected": format!("copy:{}", "c".repeat(64)), "exclusions": []}); + let context: RestoreContext = serde_json::from_value(value.clone()).unwrap(); + context.validate(Path::new("/fixture"), 3).unwrap(); + assert_eq!(serde_json::to_value(&context).unwrap(), value); + let mut invalid = context; + invalid.git = Some(GitContext { + worktree: "/fixture".into(), + prefix: String::new(), + }); + assert!(invalid.validate(Path::new("/fixture"), 4).is_err()); +} + +#[tokio::test] +async fn imported_git_preimages_support_native_keep_and_rollback_with_retained_git_context() { + use cyber_server::runtime::{ + SnapshotBinding, SnapshotOrigin, SnapshotRecoveryAction as Action, + SnapshotRecoveryConfirmation, + }; + for action in [Action::Keep, Action::Rollback] { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"target before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"recorded after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + store.restore_git_worker(prepared, None).unwrap(); + let id = store.recovery().unwrap()[0].id.clone(); + let (dir, mut journal) = store.journal(&id).unwrap(); + // Synthetic fixture origin exercises the native backend only. It grants no + // runtime/Store acknowledgement authority or worker lifetime proof. + journal.origin = Some(SnapshotOrigin { + version: 1, + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + activity_id: "op_fixture".into(), + worktree_id: None, + binding: SnapshotBinding { + directory: store.root.clone(), + root_identity: store.root_id.clone(), + namespace_identity: storage::dir_identity(&store.journals).unwrap(), + journal_id: id.clone(), + journal_identity: storage::dir_identity(&dir).unwrap(), + lock_identity: Some(store.lock_identity().unwrap()), + }, + }); + Store::record(&dir, &journal).unwrap(); + std::fs::write(repo.root.join("owner-file"), b"later owner").unwrap(); + let review = store.recovery_review(&id, action).unwrap(); + assert!(review.conflicts.is_empty(), "{:?}", review.conflicts); + let confirmation = SnapshotRecoveryConfirmation { + journal_id: id, + action, + confirmation: review.confirmation.unwrap(), + request_id: "git-native-fixture".into(), + }; + let receipt = store + .resolve_recovery(&confirmation, &|| Ok(()), None) + .unwrap(); + let (_, resolution) = store.journal(&receipt.resolution_id).unwrap(); + assert_eq!(resolution.version, 4); + assert_eq!(resolution.context.as_ref().unwrap().target, target); + assert_eq!( + std::fs::read(repo.root.join("file")).unwrap(), + if action == Action::Keep { + b"target before".as_slice() + } else { + b"recorded after".as_slice() + } + ); + assert_eq!( + std::fs::read(repo.root.join("owner-file")).unwrap(), + b"later owner" + ); + std::fs::write(repo.root.join("file"), b"owner after resolution").unwrap(); + assert_eq!( + store + .resolve_recovery(&confirmation, &|| Ok(()), None) + .unwrap(), + receipt + ); + assert_eq!( + std::fs::read(repo.root.join("file")).unwrap(), + b"owner after resolution" + ); + } +} + +#[tokio::test] +async fn git_image_handoff_enforces_byte_bounds_without_source_effects() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"bounded image").unwrap(); + let snapshot = repo.track(&repo.root).await; + let request = SnapshotRestoreRequest { + directory: repo.root.to_str().unwrap().into(), + target: snapshot.clone(), + recorded: snapshot.clone(), + expected: snapshot, + paths: None, + }; + let shadow = repo.snapshots.shadow(&request.directory).unwrap(); + assert!(shadow.prepare_restore(&request, 65536, 1).await.is_err()); + assert_eq!( + std::fs::read(repo.root.join("file")).unwrap(), + b"bounded image" + ); + assert!(repo.store(&repo.root).recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn native_git_import_rechecks_the_handed_off_blob_identity_before_reading_source() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let mut prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + prepared.target.images.get_mut("file").unwrap().bytes = b"substituted".to_vec(); + let store = repo.store(&repo.root); + assert!( + matches!(store.restore_git_worker(prepared, None), Err(RestoreError::Failed(error)) if error.contains("object identity")) + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn git_handoff_limits_changes_to_exact_reviewed_paths_inside_the_location() { + let repo = Repo::new(); + let directory = repo.root.join("[ab]"); + std::fs::create_dir(&directory).unwrap(); + for name in ["first", "second"] { + std::fs::write(directory.join(name), b"before").unwrap(); + } + let target = repo.track(&directory).await; + for name in ["first", "second"] { + std::fs::write(directory.join(name), b"after").unwrap(); + } + let recorded = repo.track(&directory).await; + let mut request = SnapshotRestoreRequest { + directory: directory.to_str().unwrap().into(), + target, + recorded: recorded.clone(), + expected: recorded, + paths: Some(vec!["[ab]/first".into()]), + }; + let shadow = repo.snapshots.shadow(&request.directory).unwrap(); + let prepared = shadow + .prepare_restore(&request, 65536, 8 * 1024 * 1024) + .await + .unwrap(); + let store = repo.store(&directory); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["first"]); + assert_eq!(std::fs::read(directory.join("first")).unwrap(), b"before"); + assert_eq!(std::fs::read(directory.join("second")).unwrap(), b"after"); + request.paths = Some(vec!["[absolutely]/first".into()]); + assert!( + shadow + .prepare_restore(&request, 65536, 8 * 1024 * 1024) + .await + .is_err() + ); +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 3fdf61666..4a70165ae 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,4 +1,5 @@ //! Private BLAKE3 snapshots for exact non-Git Locations; no user repository required. +mod git; mod ownership; mod preparation; mod resolution; diff --git a/crates/cyber-snapshot/src/copy/resolution_tests.rs b/crates/cyber-snapshot/src/copy/resolution_tests.rs index 0b9bb0f30..cf17ff093 100644 --- a/crates/cyber-snapshot/src/copy/resolution_tests.rs +++ b/crates/cyber-snapshot/src/copy/resolution_tests.rs @@ -143,6 +143,7 @@ fn reconciliation_rejects_changed_intent_and_unverifiable_legacy_writes() { ); let (dir, mut journal) = p.store.journal(&receipt.resolution_id).unwrap(); journal.context = Some(super::super::RestoreContext { + git: None, target: format!("copy:{}", "0".repeat(64)), recorded: format!("copy:{}", "0".repeat(64)), expected: format!("copy:{}", "0".repeat(64)), @@ -245,6 +246,7 @@ async fn immutable_intent_handoff_rejects_changed_context_with_unchanged_files_a guard.verify(Some(&origin)).unwrap(); let snapshot = format!("copy:{}", "0".repeat(64)); journal.context = Some(super::super::RestoreContext { + git: None, target: snapshot.clone(), recorded: snapshot.clone(), expected: snapshot, diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 57cc7d61d..1e667ea71 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -53,6 +53,8 @@ pub(super) struct RestoreContext { pub recorded: String, pub expected: String, pub exclusions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub git: Option, } impl Store { @@ -86,8 +88,13 @@ impl Store { .map_err(fail)?; if !matches!( (record.version, record.origin.is_some()), - (1, false) | (2, true) | (3, _) - ) || record.location != self.root + (1, false) | (2, true) | (3, _) | (4, _) + ) || (record.version == 4 + && !record + .context + .as_ref() + .is_some_and(|context| context.git.is_some())) + || record.location != self.root || record.root != self.root_id || record.changes.len() > ENTRY_LIMIT || record.applied.len() > record.changes.len() @@ -109,7 +116,7 @@ impl Store { .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') }) || (record.version < 3 && record.ownership.is_some()) - || (record.version == 3 + || (record.version >= 3 && record.ownership.is_none() && (record.completed || !record.applied.is_empty())) { @@ -122,14 +129,7 @@ impl Store { resolution.validate(id, &record)?; } if let Some(context) = &record.context { - for id in [&context.target, &context.recorded, &context.expected] { - if !id.strip_prefix("copy:").is_some_and(storage::valid_digest) { - return Err("Invalid recovery snapshot context".into()); - } - } - if context.exclusions.len() > ENTRY_LIMIT * 3 { - return Err("Recovery exclusion inventory exceeds limit".into()); - } + context.validate(&self.root, record.version)?; } if let Some(origin) = &record.origin { let binding = &origin.binding; @@ -183,6 +183,7 @@ impl Store { recorded: recorded.into(), expected: expected.into(), exclusions: vec![], + git: None, }; self.ensure_recovered().map_err(RestoreError::Failed)?; let before = self.capture().map_err(RestoreError::Failed)?; @@ -271,7 +272,7 @@ impl Store { } Ok((before, changes, context)) } - fn merged( + pub(super) fn merged( &self, current: Option<&Entry>, base: Option<&Entry>, @@ -485,8 +486,16 @@ impl Store { if let Some(resolution) = &mut resolution { resolution.outcome = Some(resolution.review.budget()); } + let version = if context + .as_ref() + .is_some_and(|context| context.git.is_some()) + { + 4 + } else { + 3 + }; let mut journal = Journal { - version: 3, + version, location: self.root.clone(), root: self.root_id.clone(), created: now(), diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 54b7f9f69..049a1f6f0 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -10,6 +10,8 @@ mod catalog; #[path = "git_history.rs"] pub(crate) mod history; mod images; +#[cfg(any(target_os = "macos", target_os = "linux"))] +pub(crate) mod prepared; /// A shadow git directory bound to one worktree and scoped to one Location. #[derive(Debug, Clone)] diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs new file mode 100644 index 000000000..bb94b1e9f --- /dev/null +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -0,0 +1,134 @@ +//! Immutable Git objects handed to the synchronous native journal worker. +use super::{Shadow, history, images}; +use cyber_server::runtime::SnapshotRestoreRequest; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Image { + pub mode: String, + pub object: String, + pub bytes: Vec, +} + +impl Image { + pub(crate) fn verify(&self) -> Result<(), String> { + images::verify(&self.object, &self.bytes) + } +} + +pub(crate) struct PreparedTree { + pub id: String, + /// Paths relative to the exact Location, rather than the Git worktree root. + pub images: BTreeMap, +} + +pub(crate) struct PreparedRestore { + pub directory: PathBuf, + pub worktree: PathBuf, + pub prefix: String, + pub target: PreparedTree, + pub recorded: PreparedTree, + pub expected: PreparedTree, + pub exclusions: Vec, + pub allowed: Option>, +} + +impl Shadow { + /// No source reads, Git index writes or merge subprocesses occur here. + pub(crate) async fn prepare_restore( + &self, + request: &SnapshotRestoreRequest, + max_file: u64, + max_total: u64, + ) -> Result { + let prefix = if self.pathspec == "." { + "" + } else { + &self.pathspec + }; + let directory = self.worktree.join(prefix); + if directory != std::path::Path::new(&request.directory) { + return Err("Git image handoff belongs to another Location".into()); + } + let ids = [&request.target, &request.recorded, &request.expected]; + let mut excluded = BTreeSet::new(); + for id in ids { + excluded.extend(self.recorded_exclusions(id).await?.unwrap_or_default()); + } + let exclusions = self + .historical_exclusions(&ids.map(|id| id.to_owned())) + .await?; + let allowed = request + .paths + .as_ref() + .map(|paths| { + paths + .iter() + .map(|path| relative(prefix, path)) + .collect::>() + }) + .transpose()?; + let mut remaining = max_total; + let mut trees = Vec::new(); + for id in ids { + let mut entries = BTreeMap::new(); + for (path, (mode, object)) in self.image_catalog(id).await?.iter() { + if history::excludes(&excluded, path) { + continue; + } + if !matches!(mode.as_str(), "100644" | "100755" | "120000") { + return Err("Unsupported Git journal image kind".into()); + } + if entries.len() >= 4096 { + return Err("Git journal image count exceeds limit".into()); + } + let limit = usize::try_from(max_file.min(remaining).min(64 * 1024 * 1024)) + .map_err(|_| "Git journal image limit overflow")?; + let bytes = self + .run_bounded(&["cat-file", "blob", object], limit) + .await?; + images::verify(object, &bytes)?; + remaining = remaining + .checked_sub(bytes.len() as u64) + .ok_or("Git journal image handoff exceeds limit")?; + entries.insert( + relative(prefix, path)?, + Image { + mode: mode.clone(), + object: object.clone(), + bytes, + }, + ); + } + trees.push(PreparedTree { + id: id.clone(), + images: entries, + }); + } + let mut trees = trees.into_iter(); + Ok(PreparedRestore { + directory, + worktree: self.worktree.clone(), + prefix: prefix.to_owned(), + target: trees.next().ok_or("Missing target images")?, + recorded: trees.next().ok_or("Missing recorded images")?, + expected: trees.next().ok_or("Missing expected images")?, + exclusions, + allowed, + }) + } +} + +fn relative(prefix: &str, path: &str) -> Result { + images::validate_path(path)?; + let path = if prefix.is_empty() { + path + } else { + path.strip_prefix(prefix) + .and_then(|path| path.strip_prefix('/')) + .ok_or("Git image path is outside the Location")? + }; + images::validate_path(path)?; + Ok(path.to_owned()) +} diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 818fa8017..a6ac7f5f9 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -431,11 +431,24 @@ impl Snapshots for GitSnapshots { } #[cfg(any(target_os = "macos", target_os = "linux"))] { - if !self.non_git(&request.directory) { + // Keep the same admission gate for direct port callers. The Git + // journal bridge cannot be activated before its remaining gates. + if !self.owned_restore_available(&request.directory) { return Err(RestoreError::Failed( - "Owned file-copy restore is unavailable for this Location".into(), + "Owned snapshot restore is unavailable for this Location".into(), )); } + self.recovery_location(&request.directory) + .map_err(RestoreError::Failed)?; + let shadow = if self.non_git(&request.directory) { + None + } else { + Some(self.shadow(&request.directory).ok_or_else(|| { + RestoreError::Failed( + "Owned snapshot restore is unavailable for this Location".into(), + ) + })?) + }; let config = (self.config)(Path::new(&request.directory)); let (ignore, max_file) = Self::options_value(&config); let max_total = config @@ -448,6 +461,15 @@ impl Snapshots for GitSnapshots { max_total, private_roots: self.private_roots.clone(), }; + let prepared = match shadow { + Some(shadow) => Some( + shadow + .prepare_restore(&request, options.max_file, options.max_total) + .await + .map_err(RestoreError::Failed)?, + ), + None => None, + }; let data = self.data_dir.clone(); let lock = cyber_server::workspace::mutation_lock(Path::new(&request.directory)) .map_err(|error| RestoreError::Failed(error.to_string()))?; @@ -456,14 +478,18 @@ impl Snapshots for GitSnapshots { .run(move |work| { let store = copy::Store::open(&data, &request.directory, options) .map_err(RestoreError::Failed)?; - let paths = store.restore_worker( - &request.target, - &request.recorded, - &request.expected, - request.paths.as_deref(), - &|| Ok(()), - Some(work), - )?; + let paths = if let Some(prepared) = prepared { + store.restore_git_worker(prepared, Some(work))? + } else { + store.restore_worker( + &request.target, + &request.recorded, + &request.expected, + request.paths.as_deref(), + &|| Ok(()), + Some(work), + )? + }; let origin = work.origin().cloned(); let digest = store .worker_digest(origin.as_ref()) diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index f4124cda8..fd608f2c2 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1738,3 +1738,64 @@ bytes; legacy bare trees retain their exact supplied identity. This closes the substitution window rather than trusting a subsequent rev-parse result. Final formatting, generated SDK consistency and diff checks passed. Strict pinned OpenSpec passed 117 items; spec lint reports 0 errors/25 warnings. + + +## Git image import into the native journal (2026-11-07) + +`git/prepared.rs` now constructs a bounded image packet from verified private Git +objects before the synchronous source callback. It retains original target, recorded +and expected IDs, leaf IDs/modes/bytes, exact worktree/Location prefix, allowed paths +and the union of historical exclusions. Paths become exact Location-relative names +for the native engine. Per-file output and combined packet bytes are bounded; each +tree has at most 4,096 images. No source reads, merge subprocesses or Git index +writes occur in this object preparation. Native import verifies the handed-off leaf +hashes again and revalidates every known expected path before source replacement. + +`copy/git.rs` imports regular files through the existing CAS, conflict merge, native +source/stage/parent ownership and durable journal executor. Version-4 intents retain +a distinct exact Git context with the original IDs; no copy IDs are manufactured. +Git contexts reject wrong versions, mixed/invalid identities and foreign Locations. +The complete-record ownership requirement applies to version 4 as well as version 3. +Legacy copy contexts omit the new optional field, preserving their encoded identity. +The owned adapter has the preparation/worker dispatch, but its ordinary Git +availability and lifetime reservation gates remain disabled. There is no new +production admission claim for this partial integration. + +Nine added tests cover real Git image capture followed by native regular-file +creation/deletion/restoration and executable modes, retained displaced preimages, +user index preservation, owner-edit merging, stale handoff refusal, exact glob-shaped +subdirectory scope, historical ignored-file protection, corrupt handoff bytes, +packet byte bounds and legacy context compatibility. A positive test removes the +shadow repository after object preparation; native execution still succeeds without +Git access. Native fixture keep/rollback reuses those actual preimages, retains Git +context in its independent resolution intent and returns the same request receipt +after a later owner edit without replay. Fixture origins are synthetic and provide +no runtime acknowledgement authority or process-lifetime evidence. A separate +negative test refuses symlink images before source or journal effects; that refusal +is explicitly not symlink compatibility acceptance. + +The next work is symbolic-link image/native ownership support, complete expected +state scanning (including new paths and Git ignore semantics), actual Git worker +lifetime/reservation and runtime acknowledgement integration, then process-kill +acceptance. Current native import does not yet handle file/directory transitions or +literal Unix backslashes through the copy engine. These requirements prevent +enabling the Git gate. Full recovery cleanup, clients and platform acceptance also +remain open. No parent task is accepted by the regular-file fixtures. + +Code review covered all changed adapter, codec, native import, tests and spec files. +The existing snapshot adapter/native engine split is preserved; no unrelated +architecture was introduced. Review required typed hash revalidation at import, +version-4 context/ownership refusal and enforcement of the availability gate inside +the direct owned port as well as its caller. These corrections are included. An +additional exact reviewed-path test restores only one file and refuses a similar +foreign scope prefix. No unresolved critical/major finding remains for this +disabled regular-file bridge; the activation gaps above remain mandatory. + +Verification on local macOS arm64: locked snapshot tests passed 103 cases, with +two ignored subprocess fixtures (the pathspec fixture is invoked by its parent). +All-target snapshot Clippy passed with warnings denied after the import and direct +gate changes. Formatting, generated SDK consistency and diff checks passed. Strict +pinned OpenSpec passed all 117 items; spec lint reports 0 errors/25 warnings. Full +workspace tests, SDK execution, real CLI/TUI/App acceptance, native process-kill +Git acceptance and Linux/Windows execution were not run for this increment. The +backend fixture keep/rollback tests are not runtime lifecycle or crash evidence. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 5cdec9681..66826041e 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -467,3 +467,17 @@ nested tree cannot make a restore appear to have no changes. Metadata reads are bounded and over-limit subprocess output is killed and awaited. This catalog is preparation input for the future Git journal bridge, not native origin authority; source mutation must still run under the retained synchronous filesystem worker. + + +### Git journal handoff + +Git object preparation may read only immutable private snapshot objects before the +synchronous native source callback. The bounded handoff retains original target, +recorded and expected Git identities, exact worktree/Location prefix, historical +exclusions and verified images; it must not manufacture copy snapshot identities. +Native import reads/revalidates source only inside the existing retained worker and +uses the same durable change/preimage/ownership journal. A distinct journal context +version binds Git identity and Location. Legacy copy contexts keep their existing +encoded identity. Git admission remains disabled until symbolic links, full expected +state revalidation and all native recovery/acknowledgement paths are integrated; +regular-file import tests alone do not authorize that switch. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index e2e37d5ed..2cdc052ba 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -137,6 +137,13 @@ - **AND** exclusion tags and every scoped directory tree SHALL match their object IDs before changed paths are derived from a verified literal-path catalog - **AND** metadata traversal and subprocess output SHALL be bounded, with over-limit children killed and awaited +#### Scenario: Git images enter a native recovery journal +- **WHEN** verified Git images are imported by the retained native snapshot worker +- **THEN** the journal SHALL retain original target/recorded/expected Git identities and exact worktree/Location context in a distinct versioned context +- **AND** the bounded handoff SHALL retain and revalidate each leaf object identity before native source planning, without Git subprocesses inside the synchronous source callback +- **AND** native source changes SHALL retain displaced preimages and use the same ownership, recovery review and retry-lookup rules as file-copy journals +- **AND** legacy copy contexts SHALL preserve their encoded identity and SHALL NOT be reinterpreted as Git contexts + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index c86f0d93a..67111719b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -268,3 +268,15 @@ and glob-shaped names independently of inherited pathspec environment. Real corrupt tag/root/nested-tree restores refuse without changing source or user index; scoped restores preserve sibling files. This remains preparation for owned Git journals; M2–M8 and parent tasks are unchanged and incomplete. + + +Git/native regular-file handoff continuation: the owned adapter now prepares a +bounded immutable Git object packet before the synchronous filesystem callback. +Native import rechecks leaf hashes and known expected paths, uses the shared CAS/ +preimage/ownership engine and writes version-4 intents with original Git IDs and +exact worktree/Location context. Native fixture keep/rollback and request retry +retain that context; legacy copy encoding and fences remain distinct. Ordinary +Git worker reservation/admission is still disabled. Symbolic links, full expected +state scanning (including newly appearing paths), exact runtime Git lifetime/ +receipt acknowledgement, retention and complete crash/client/platform acceptance +remain required; M2–M8 and parent tasks remain open. From 0cb9eeb0690264ed62d045e03eb71cc3044fc51c Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 03:01:19 -0400 Subject: [PATCH 22/87] Journal symbolic-link images without following their targets --- crates/cyber-core/src/binary/native.rs | 10 + crates/cyber-server/src/runtime/mod.rs | 5 +- .../src/runtime/snapshot_recovery_types.rs | 36 ++ crates/cyber-snapshot/src/copy/git.rs | 58 ++- crates/cyber-snapshot/src/copy/git/tests.rs | 191 ++++++++- crates/cyber-snapshot/src/copy/links.rs | 381 ++++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 33 +- crates/cyber-snapshot/src/copy/ownership.rs | 45 ++- .../src/copy/ownership_tests.rs | 115 +++++- crates/cyber-snapshot/src/copy/resolution.rs | 50 ++- crates/cyber-snapshot/src/copy/restore.rs | 132 +++++- crates/cyber-snapshot/src/copy/tree.rs | 9 + docs/rewind.md | 9 + .../implementation.md | 84 ++++ .../managed-recovery.md | 16 + .../specs/snapshots-checkpoints/spec.md | 8 + .../complete-daily-coding-surfaces/tasks.md | 12 + sdk/openapi.json | 10 + sdk/typescript/src/generated/types.ts | 3 + 19 files changed, 1149 insertions(+), 58 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/links.rs diff --git a/crates/cyber-core/src/binary/native.rs b/crates/cyber-core/src/binary/native.rs index 2dda791f7..41c4d5e0c 100644 --- a/crates/cyber-core/src/binary/native.rs +++ b/crates/cyber-core/src/binary/native.rs @@ -121,6 +121,16 @@ pub fn plain_snapshot_directory(file: &File) -> Result { plain_snapshot(file, metadata_fields(file, &m)?) } +/// Plain metadata from a descriptor opened on the link itself, never its target. +#[cfg(target_os = "macos")] +pub fn plain_snapshot_symlink(file: &File) -> Result { + let m = file.metadata().map_err(|_| Error::Io)?; + if !m.file_type().is_symlink() || m.nlink() != 1 || m.mode() & 0o7000 != 0 { + return Err(Error::File); + } + plain_snapshot(file, metadata_fields(file, &m)?) +} + fn plain_snapshot(file: &File, value: Metadata) -> Result { if !attributes(file)?.is_empty() { return Err(Error::File); diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 3bf95558d..3b1385486 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -144,8 +144,9 @@ pub use rewind_preview::{ }; pub use snapshot_recovery_types::{ SnapshotRecoveryAction, SnapshotRecoveryConfirmation, SnapshotRecoveryImage, - SnapshotRecoveryPath, SnapshotRecoveryReceipt, SnapshotRecoveryResult, SnapshotRecoveryReview, - SnapshotRecoverySelection, SnapshotRecoveryState, + SnapshotRecoveryImageKind, SnapshotRecoveryPath, SnapshotRecoveryReceipt, + SnapshotRecoveryResult, SnapshotRecoveryReview, SnapshotRecoverySelection, + SnapshotRecoveryState, }; pub use snapshot_work::{ SnapshotBinding, SnapshotCompletion, SnapshotNativeGuard, SnapshotNativeResult, SnapshotOrigin, diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs index 286ddaec1..ad39415e7 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs @@ -8,8 +8,25 @@ pub enum SnapshotRecoveryAction { Rollback, } +#[derive( + Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum SnapshotRecoveryImageKind { + #[default] + File, + Symlink, +} +impl SnapshotRecoveryImageKind { + fn is_file(&self) -> bool { + *self == Self::File + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] pub struct SnapshotRecoveryImage { + #[serde(default, skip_serializing_if = "SnapshotRecoveryImageKind::is_file")] + pub kind: SnapshotRecoveryImageKind, pub hash: String, pub bytes: u64, pub mode: u32, @@ -111,3 +128,22 @@ pub struct SnapshotRecoveryResult { /// has not been observed by this response, even if runtime settlement is durable. pub journal_acknowledged: Option, } + +#[cfg(test)] +mod image_tests { + use super::*; + #[test] + fn legacy_recovery_images_default_to_files_and_link_images_roundtrip_explicitly() { + let old = serde_json::json!({"hash":"fixture", "bytes":3, "mode":420, "uid":1,"gid":2,"native_identity":null}); + let mut image: SnapshotRecoveryImage = serde_json::from_value(old.clone()).unwrap(); + assert_eq!(image.kind, SnapshotRecoveryImageKind::File); + assert_eq!(serde_json::to_value(&image).unwrap(), old); + image.kind = SnapshotRecoveryImageKind::Symlink; + let encoded = serde_json::to_value(&image).unwrap(); + assert_eq!(encoded["kind"], "symlink"); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + image + ); + } +} diff --git a/crates/cyber-snapshot/src/copy/git.rs b/crates/cyber-snapshot/src/copy/git.rs index 0b3c48799..9a98ffeca 100644 --- a/crates/cyber-snapshot/src/copy/git.rs +++ b/crates/cyber-snapshot/src/copy/git.rs @@ -20,7 +20,7 @@ impl RestoreContext { pub(super) fn validate(&self, location: &Path, version: u32) -> Result<()> { let ids = [&self.target, &self.recorded, &self.expected]; if let Some(git) = &self.git { - if version != 4 + if !matches!(version, 4 | 5) || !git.worktree.is_absolute() || git .worktree @@ -59,7 +59,13 @@ fn matches(entry: Option<&Entry>, image: Option<&Image>) -> bool { (Some(entry), Some(image)) => { entry.length == image.bytes.len() as u64 && entry.object == storage::digest(&image.bytes) - && (entry.mode & 0o111 != 0) == (image.mode == "100755") + && match entry.kind { + super::ImageKind::File => { + image.mode != "120000" + && (entry.mode & 0o111 != 0) == (image.mode == "100755") + } + super::ImageKind::Symlink => image.mode == "120000", + } } _ => false, } @@ -67,7 +73,7 @@ fn matches(entry: Option<&Entry>, image: Option<&Image>) -> bool { impl Store { /// Called only inside a retained synchronous worker. Ordinary Git admission - /// stays disabled until link import and full expected-state scanning are ready. + /// stays disabled until full state/lifetime and native acceptance gates are ready. pub(crate) fn restore_git_worker( &self, prepared: PreparedRestore, @@ -110,17 +116,24 @@ impl Store { )); } self.ensure_recovered().map_err(RestoreError::Failed)?; - // Known image validation precedes any source effect. Link support is required - // before enabling the runtime gate; this import currently handles files. + // Known image validation precedes any source effect. Full state scanning + // and actual runtime lifetime integration remain required before activation. for tree in [&prepared.target, &prepared.recorded, &prepared.expected] { for (path, image) in &tree.images { image.verify().map_err(RestoreError::Failed)?; self.check_source_path(path).map_err(RestoreError::Failed)?; - if !matches!(image.mode.as_str(), "100644" | "100755") { + if !matches!(image.mode.as_str(), "100644" | "100755" | "120000") { return Err(RestoreError::Failed( - "Native Git link import is not yet available".into(), + "Unsupported native Git image mode".into(), )); } + if image.mode == "120000" + && (image.bytes.is_empty() + || image.bytes.len() > super::links::TARGET_LIMIT + || image.bytes.contains(&0)) + { + return Err(RestoreError::Failed("Invalid symbolic-link target".into())); + } if image.bytes.len() as u64 > self.options.max_file { return Err(RestoreError::Failed( "Git journal image exceeds file limit".into(), @@ -183,6 +196,16 @@ impl Store { } } }; + if let Some(entry) = &next + && entry.kind == super::ImageKind::Symlink + { + self.check_link_target( + &path, + &self.bytes(entry).map_err(RestoreError::Failed)?, + &prepared.worktree, + ) + .map_err(RestoreError::Failed)?; + } changes.push(Change { path, before: current, @@ -204,12 +227,23 @@ impl Store { .into_std_file() .metadata() .map_err(storage::fail)?; - entry.mode = current.map_or(0o644, |entry| entry.mode); - entry.mode = if image.mode == "100755" { - entry.mode | 0o111 + if image.mode == "120000" { + entry.kind = super::ImageKind::Symlink; + entry.mode = current + .filter(|entry| entry.kind == super::ImageKind::Symlink) + .map_or(0o777, |entry| entry.mode); } else { - entry.mode & !0o111 - }; + entry.mode = current + .filter(|entry| entry.kind == super::ImageKind::File) + .map_or(0o644, |entry| entry.mode); + } + if entry.kind == super::ImageKind::File { + entry.mode = if image.mode == "100755" { + entry.mode | 0o111 + } else { + entry.mode & !0o111 + }; + } entry.uid = current.map_or(owner.uid(), |entry| entry.uid); entry.gid = current.map_or(owner.gid(), |entry| entry.gid); Ok(entry) diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index 9e5dc45db..c61bd721b 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -144,7 +144,7 @@ async fn native_git_import_restores_files_modes_creations_deletions_and_retains_ let journals = store.recovery().unwrap(); assert_eq!(journals.len(), 1); let (dir, journal) = store.journal(&journals[0].id).unwrap(); - assert_eq!(journal.version, 4); + assert_eq!(journal.version, 5); let context = journal.context.as_ref().unwrap(); assert_eq!( (&context.target, &context.recorded, &context.expected), @@ -159,6 +159,11 @@ async fn native_git_import_restores_files_modes_creations_deletions_and_retains_ .snapshots .owned_restore_available(repo.root.to_str().unwrap()) ); + let mut legacy_git = journal.clone(); + legacy_git.version = 4; + legacy_git.backend = None; + Store::record(&dir, &legacy_git).unwrap(); + assert!(store.journal(&journals[0].id).is_ok()); let mut downgraded = journal.clone(); downgraded.version = 3; Store::record(&dir, &downgraded).unwrap(); @@ -255,9 +260,9 @@ async fn native_git_import_keeps_subdirectory_and_historical_exclusion_boundarie } #[tokio::test] -async fn native_git_import_refuses_link_images_before_any_journal_or_source_effect() { +async fn native_git_import_refuses_escaping_links_before_any_journal_or_source_effect() { let repo = Repo::new(); - std::os::unix::fs::symlink("destination", repo.root.join("link")).unwrap(); + std::os::unix::fs::symlink("../destination", repo.root.join("link")).unwrap(); let target = repo.track(&repo.root).await; std::fs::remove_file(repo.root.join("link")).unwrap(); std::fs::write(repo.root.join("link"), b"current owner").unwrap(); @@ -267,7 +272,7 @@ async fn native_git_import_refuses_link_images_before_any_journal_or_source_effe .await; let store = repo.store(&repo.root); assert!( - matches!(store.restore_git_worker(prepared, None), Err(RestoreError::Failed(error)) if error.contains("link import")) + matches!(store.restore_git_worker(prepared, None), Err(RestoreError::Failed(error)) if error.contains("outside the worktree")) ); assert!(store.recovery().unwrap().is_empty()); assert_eq!( @@ -341,7 +346,7 @@ async fn imported_git_preimages_support_native_keep_and_rollback_with_retained_g .resolve_recovery(&confirmation, &|| Ok(()), None) .unwrap(); let (_, resolution) = store.journal(&receipt.resolution_id).unwrap(); - assert_eq!(resolution.version, 4); + assert_eq!(resolution.version, 5); assert_eq!(resolution.context.as_ref().unwrap().target, target); assert_eq!( std::fs::read(repo.root.join("file")).unwrap(), @@ -446,3 +451,179 @@ async fn git_handoff_limits_changes_to_exact_reviewed_paths_inside_the_location( .is_err() ); } + +/// Synthetic native fixture only; no runtime acknowledgement authority is granted. +fn own_native_journal(store: &Store, id: &str) { + use cyber_server::runtime::{SnapshotBinding, SnapshotOrigin}; + let (dir, mut journal) = store.journal(id).unwrap(); + journal.origin = Some(SnapshotOrigin { + version: 1, + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + activity_id: "op_fixture".into(), + worktree_id: None, + binding: SnapshotBinding { + directory: store.root.clone(), + root_identity: store.root_id.clone(), + namespace_identity: storage::dir_identity(&store.journals).unwrap(), + journal_id: id.into(), + journal_identity: storage::dir_identity(&dir).unwrap(), + lock_identity: Some(store.lock_identity().unwrap()), + }, + }); + Store::record(&dir, &journal).unwrap(); +} +fn put_node(root: &Path, node: &str) { + let path = root.join("entry"); + if std::fs::symlink_metadata(&path).is_ok() { + std::fs::remove_file(&path).unwrap(); + } + match node { + "absent" => {} + "file" => std::fs::write(path, b"regular owner bytes").unwrap(), + target => std::os::unix::fs::symlink(target, path).unwrap(), + } +} +fn assert_node(root: &Path, node: &str) { + let path = root.join("entry"); + match node { + "absent" => assert!(std::fs::symlink_metadata(path).is_err()), + "file" => assert_eq!(std::fs::read(path).unwrap(), b"regular owner bytes"), + target => assert_eq!(std::fs::read_link(path).unwrap(), Path::new(target)), + } +} + +#[tokio::test] +async fn native_git_link_changes_and_rollback_never_follow_target_files() { + use cyber_server::runtime::{SnapshotRecoveryAction as Action, SnapshotRecoveryConfirmation}; + for (want, recorded_node) in [ + ("one", "two"), + ("one", "file"), + ("file", "two"), + ("one", "absent"), + ("absent", "two"), + ("missing/child", "file"), + ] { + let repo = Repo::new(); + std::fs::write(repo.root.join("one"), b"first target owner data").unwrap(); + std::fs::write(repo.root.join("two"), b"second target owner data").unwrap(); + put_node(&repo.root, want); + let target = repo.track(&repo.root).await; + put_node(&repo.root, recorded_node); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert_eq!( + store.restore_git_worker(prepared, None).unwrap(), + ["entry"], + "{want}/{recorded_node}" + ); + assert_node(&repo.root, want); + let id = store.recovery().unwrap()[0].id.clone(); + own_native_journal(&store, &id); + let review = store.recovery_review(&id, Action::Rollback).unwrap(); + assert!(review.conflicts.is_empty(), "{:?}", review.conflicts); + let desired = review + .paths + .iter() + .find(|path| path.path == "entry") + .unwrap(); + if want != "file" && want != "absent" { + assert_eq!( + desired.current.as_ref().unwrap().kind, + super::super::ImageKind::Symlink + ); + } + let confirmation = SnapshotRecoveryConfirmation { + journal_id: id, + action: Action::Rollback, + confirmation: review.confirmation.unwrap(), + request_id: "link-rollback".into(), + }; + let receipt = store + .resolve_recovery(&confirmation, &|| Ok(()), None) + .unwrap(); + assert_node(&repo.root, recorded_node); + assert_eq!( + std::fs::read(repo.root.join("one")).unwrap(), + b"first target owner data" + ); + assert_eq!( + std::fs::read(repo.root.join("two")).unwrap(), + b"second target owner data" + ); + let (_, journal) = store.journal(&receipt.resolution_id).unwrap(); + assert_eq!(journal.version, 5); + std::fs::write(repo.root.join("later-owner"), b"owner after resolution").unwrap(); + assert_eq!( + store + .resolve_recovery(&confirmation, &|| Ok(()), None) + .unwrap(), + receipt + ); + } +} + +#[tokio::test] +async fn native_git_link_target_aliases_cannot_escape_or_enter_protected_namespaces() { + for target_name in [ + "alias/secret", + "alias/../secret", + ".git/config", + ".GIT/config", + "../outside/secret", + ] { + let repo = Repo::new(); + if target_name == ".GIT/config" && !repo.root.join(target_name).exists() { + continue; + } + let outside = repo.root.parent().unwrap().join("outside"); + std::fs::create_dir(&outside).unwrap(); + std::fs::write(outside.join("secret"), b"private target data").unwrap(); + std::fs::create_dir(outside.join("child")).unwrap(); + std::os::unix::fs::symlink(outside.join("child"), repo.root.join("alias")).unwrap(); + put_node(&repo.root, target_name); + let target = repo.track(&repo.root).await; + put_node(&repo.root, "file"); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_node(&repo.root, "file"); + assert!(store.recovery().unwrap().is_empty()); + assert_eq!( + std::fs::read(outside.join("secret")).unwrap(), + b"private target data" + ); + } +} + +#[tokio::test] +async fn native_link_targets_cannot_use_case_aliases_of_configured_private_roots() { + let repo = Repo::new(); + let private = repo.root.join("Private"); + std::fs::create_dir(&private).unwrap(); + if !repo.root.join("PRIVATE").exists() { + return; + } + put_node(&repo.root, "PRIVATE/missing"); + let target = repo.track(&repo.root).await; + put_node(&repo.root, "file"); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let mut store = repo.store(&repo.root); + store.options.private_roots.push(private); + let result = store.restore_git_worker(prepared, None); + assert!( + matches!(&result, Err(RestoreError::Failed(error)) if error.contains("private root") || error.contains("protected")), + "{result:?}" + ); + assert_node(&repo.root, "file"); + assert!(store.recovery().unwrap().is_empty()); +} diff --git a/crates/cyber-snapshot/src/copy/links.rs b/crates/cyber-snapshot/src/copy/links.rs new file mode 100644 index 000000000..831b483f3 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/links.rs @@ -0,0 +1,381 @@ +//! Link-inode observations. No file descriptor here follows a link target. +use super::storage::{self, Result, fail}; +use super::{Entry, ImageKind, Store}; +use cap_std::fs::Dir; +use std::ffi::OsStr; +use std::fs::File; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::MetadataExt; +#[cfg(target_os = "macos")] +use std::os::unix::fs::PermissionsExt; +use std::path::{Component, Path, PathBuf}; + +pub(super) const TARGET_LIMIT: usize = 4096; + +pub(super) fn pin_image(dir: &Dir, name: &OsStr) -> Result { + if dir + .symlink_metadata(name) + .map_err(fail)? + .file_type() + .is_symlink() + { + pin_link(dir, name) + } else { + storage::open(dir, Path::new(name)) + } +} +fn pin_link(dir: &Dir, name: &OsStr) -> Result { + if Path::new(name).file_name() != Some(name) { + return Err("Invalid symbolic-link leaf".into()); + } + // macOS O_SYMLINK opens the link itself; combining it with O_NOFOLLOW + // would reject the link instead. Linux O_PATH|O_NOFOLLOW pins its inode. + #[cfg(target_os = "macos")] + let flags = rustix::fs::OFlags::SYMLINK | rustix::fs::OFlags::NONBLOCK; + #[cfg(target_os = "linux")] + let flags = rustix::fs::OFlags::PATH | rustix::fs::OFlags::NOFOLLOW; + let file = File::from( + rustix::fs::openat( + dir, + name, + flags | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .map_err(fail)?, + ); + let metadata = file.metadata().map_err(fail)?; + if !metadata.file_type().is_symlink() || metadata.nlink() != 1 { + return Err("Snapshot source is not a unique symbolic link".into()); + } + Ok(file) +} + +/// Bound the read independently of the file's reported target length. +pub(super) fn read(dir: &Dir, name: &OsStr, limit: u64) -> Result<(File, Vec)> { + let pin = pin_link(dir, name)?; + let before = super::ownership::generation(&pin)?; + let parent = dir.try_clone().map_err(fail)?.into_std_file(); + let parent_before = super::ownership::generation(&parent)?; + plain_metadata(dir, name, &pin)?; + let mut bytes = [0u8; TARGET_LIMIT + 1]; + let size = rustix::fs::readlinkat_raw(dir, name, &mut bytes[..]).map_err(fail)?; + if size == 0 || size > TARGET_LIMIT || size as u64 > limit { + return Err("Symbolic-link target exceeds snapshot bounds".into()); + } + let current = pin_link(dir, name)?; + if storage::identity(¤t)? != storage::identity(&pin)? + || super::ownership::generation(&pin)? != before + || super::ownership::generation(¤t)? != before + || super::ownership::generation(&parent)? != parent_before + { + return Err("Symbolic-link route or metadata changed while inspecting".into()); + } + Ok((pin, bytes[..size].to_vec())) +} + +#[cfg(target_os = "macos")] +fn plain_metadata(_dir: &Dir, _name: &OsStr, pin: &File) -> Result<()> { + cyber_core::binary::native::plain_snapshot_symlink(pin) + .map(|_| ()) + .map_err(|_| "Unsupported symbolic-link metadata".into()) +} +#[cfg(target_os = "linux")] +fn plain_metadata(dir: &Dir, name: &OsStr, pin: &File) -> Result<()> { + use std::os::fd::AsRawFd; + let metadata = pin.metadata().map_err(fail)?; + if metadata.mode() & 0o777 != 0o777 { + return Err("Unsupported symbolic-link mode".into()); + } + // O_PATH descriptors do not support fgetxattr. The kernel proc route keeps + // the already pinned parent, and llistxattr never follows the leaf. Both the + // parent generation and pinned link generation are checked around this read. + let parent_route = PathBuf::from(format!("/proc/self/fd/{}", dir.as_raw_fd())); + let routed = File::open(&parent_route).map_err(fail)?; + if storage::identity(&routed)? != storage::dir_identity(dir)? { + return Err("Symbolic-link metadata route changed".into()); + } + let mut attributes = [0u8; 65536]; + if rustix::fs::llistxattr(parent_route.join(name), &mut attributes[..]).map_err(fail)? != 0 { + return Err("Unsupported symbolic-link attributes".into()); + } + Ok(()) +} + +impl Store { + pub(super) fn link_entry(&self, dir: &Dir, name: &OsStr, save: bool) -> Result { + let (pin, bytes) = read(dir, name, self.options.max_file)?; + let metadata = pin.metadata().map_err(fail)?; + let mut entry = if save { + self.object(&bytes)? + } else { + Entry { + kind: ImageKind::Symlink, + object: storage::digest(&bytes), + length: bytes.len() as u64, + mode: 0, + uid: 0, + gid: 0, + } + }; + entry.kind = ImageKind::Symlink; + entry.mode = metadata.mode() & 0o777; + entry.uid = metadata.uid(); + entry.gid = metadata.gid(); + Ok(entry) + } + + pub(super) fn stage_image( + &self, + dir: &Dir, + name: &str, + entry: &Entry, + bytes: &[u8], + ) -> Result<()> { + if entry.kind == ImageKind::File { + let file = storage::create(dir, name, bytes)?; + return Self::stage_metadata(&file, entry); + } + if bytes.is_empty() || bytes.len() > TARGET_LIMIT || bytes.contains(&0) { + return Err("Invalid symbolic-link target".into()); + } + rustix::fs::symlinkat(OsStr::from_bytes(bytes), dir, name).map_err(fail)?; + let pin = pin_link(dir, OsStr::new(name))?; + #[cfg(target_os = "macos")] + { + rustix::fs::fchown( + &pin, + Some(rustix::fs::Uid::from_raw(entry.uid)), + Some(rustix::fs::Gid::from_raw(entry.gid)), + ) + .map_err(fail)?; + pin.set_permissions(std::fs::Permissions::from_mode(entry.mode)) + .map_err(fail)?; + storage::sync_file(&pin)?; + } + #[cfg(target_os = "linux")] + { + if entry.mode != 0o777 { + return Err("Unsupported symbolic-link mode".into()); + } + rustix::fs::chownat( + dir, + name, + Some(rustix::fs::Uid::from_raw(entry.uid)), + Some(rustix::fs::Gid::from_raw(entry.gid)), + rustix::fs::AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(fail)?; + } + if storage::identity(&pin_image(dir, OsStr::new(name))?)? != storage::identity(&pin)? { + return Err("Prepared symbolic-link inode changed".into()); + } + storage::sync(dir) + } + + /// Rechecked immediately before publication. Canonicalize only an existing + /// prefix so safe dangling targets remain representable without reading data. + pub(super) fn check_link_target( + &self, + path: &str, + bytes: &[u8], + boundary: &Path, + ) -> Result<()> { + if bytes.is_empty() || bytes.len() > TARGET_LIMIT || bytes.contains(&0) { + return Err("Invalid symbolic-link target".into()); + } + let target = self + .root + .join(path) + .parent() + .ok_or("Invalid link parent")? + .join(OsStr::from_bytes(bytes)); + let mut normalized = PathBuf::new(); + for part in target.components() { + match part { + Component::ParentDir => { + if !normalized.pop() { + return Err("Unsafe symbolic-link target".into()); + } + } + Component::CurDir => {} + other => normalized.push(other), + } + } + self.check_link_boundary(&normalized, boundary)?; + // Keep the original component order: resolving alias/../file can escape + // even when a lexical collapse of the same spelling stays inside. + let mut prefix = target.as_path(); + let mut missing = Vec::new(); + loop { + match std::fs::symlink_metadata(prefix) { + Ok(_) => break, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + missing.push(prefix.file_name().ok_or("Invalid symbolic-link prefix")?); + prefix = prefix.parent().ok_or("Invalid symbolic-link prefix")?; + } + Err(error) => return Err(fail(error)), + } + } + let mut physical = prefix.canonicalize().map_err(fail)?; + for part in missing.into_iter().rev() { + physical.push(part); + } + self.check_link_boundary(&physical, boundary)?; + self.check_link_private_identities(&physical, boundary) + } + + fn check_link_private_identities(&self, target: &Path, boundary: &Path) -> Result<()> { + let mut private = std::collections::BTreeSet::new(); + for path in self + .options + .private_roots + .iter() + .chain(std::iter::once(&self.data_path)) + { + if let Some(metadata) = existing_metadata(path)? { + private.insert((metadata.dev(), metadata.ino())); + } + } + for cursor in target + .ancestors() + .take_while(|cursor| cursor.starts_with(boundary)) + { + let Some(metadata) = existing_metadata(cursor)? else { + continue; + }; + let identity = (metadata.dev(), metadata.ino()); + if private.contains(&identity) { + return Err("Symbolic-link target aliases a private root".into()); + } + if let Some(parent) = cursor.parent() { + // Do not depend on a resolver normalizing filesystem aliases. + // Compare actual nodes to reserved siblings instead of folding names. + for name in [".git", ".cyber", ".claude"] { + if existing_metadata(&parent.join(name))? + .is_some_and(|reserved| (reserved.dev(), reserved.ino()) == identity) + { + return Err("Symbolic-link target aliases a protected namespace".into()); + } + } + } + } + Ok(()) + } + fn check_link_boundary(&self, target: &Path, boundary: &Path) -> Result<()> { + let relative = target + .strip_prefix(boundary) + .map_err(|_| "Symbolic-link target is outside the worktree")?; + if super::tree::protected(relative) + || target.starts_with(&self.data_path) + || self + .options + .private_roots + .iter() + .any(|root| target.starts_with(root)) + { + return Err("Symbolic-link target is protected".into()); + } + Ok(()) + } +} + +fn existing_metadata(path: &Path) -> Result> { + match std::fs::metadata(path) { + Ok(metadata) => Ok(Some(metadata)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(fail(error)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn pinning_and_reading_a_dangling_link_never_opens_its_target_and_rejects_hardlinks() { + let temp = tempfile::tempdir().unwrap(); + let dir = Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(); + std::os::unix::fs::symlink("missing", temp.path().join("link")).unwrap(); + let (pin, bytes) = read(&dir, OsStr::new("link"), TARGET_LIMIT as u64).unwrap(); + assert!(pin.metadata().unwrap().file_type().is_symlink()); + assert_eq!(bytes, b"missing"); + assert!(!temp.path().join("missing").exists()); + assert!(read(&dir, OsStr::new("link"), 1).is_err()); + std::fs::hard_link(temp.path().join("link"), temp.path().join("alias")).unwrap(); + assert!(pin_image(&dir, OsStr::new("link")).is_err()); + } + #[test] + fn same_target_link_substitution_has_a_distinct_native_identity() { + let temp = tempfile::tempdir().unwrap(); + let dir = Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(); + std::os::unix::fs::symlink("missing", temp.path().join("link")).unwrap(); + let old = pin_image(&dir, OsStr::new("link")).unwrap(); + std::fs::rename(temp.path().join("link"), temp.path().join("retained")).unwrap(); + std::os::unix::fs::symlink("missing", temp.path().join("link")).unwrap(); + assert_ne!( + storage::identity(&old).unwrap(), + storage::identity(&pin_image(&dir, OsStr::new("link")).unwrap()).unwrap() + ); + } + #[cfg(target_os = "macos")] + #[test] + fn link_attributes_are_inspected_on_the_pinned_link_and_refuse_before_recreation() { + let temp = tempfile::tempdir().unwrap(); + let dir = Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(); + std::os::unix::fs::symlink("missing", temp.path().join("link")).unwrap(); + let pin = pin_image(&dir, OsStr::new("link")).unwrap(); + rustix::fs::fsetxattr( + &pin, + "user.snapshot-test", + b"retained metadata", + rustix::fs::XattrFlags::empty(), + ) + .unwrap(); + assert!(read(&dir, OsStr::new("link"), 4096).is_err()); + assert_eq!( + std::fs::read_link(temp.path().join("link")).unwrap(), + Path::new("missing") + ); + assert!(!temp.path().join("missing").exists()); + } + #[test] + fn protected_and_private_node_checks_do_not_depend_on_alias_spelling() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::write(root.join(".git/config"), b"fixture").unwrap(); + std::fs::create_dir(root.join("Private")).unwrap(); + if !root.join(".GIT").exists() { + return; + } + let mut store = Store::open( + &temp.path().join("data"), + root.to_str().unwrap(), + super::super::Options { + max_file: 65536, + max_total: 8 * 1024 * 1024, + ignore: vec![], + private_roots: vec![root.join("Private")], + }, + ) + .unwrap(); + // Store canonicalizes its Location; preserve aliases only in the target. + let boundary = store.root.clone(); + assert!( + store + .check_link_private_identities(&boundary.join(".GIT/config"), &boundary) + .is_err() + ); + assert!( + store + .check_link_private_identities(&boundary.join("PRIVATE/missing"), &boundary) + .is_err() + ); + store.options.private_roots.clear(); + assert!( + store + .check_link_private_identities(&boundary.join("PRIVATE/missing"), &boundary) + .is_ok() + ); + } +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 4a70165ae..d5b8f639a 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,5 +1,6 @@ -//! Private BLAKE3 snapshots for exact non-Git Locations; no user repository required. +//! Location-bound native recovery journals and private non-Git BLAKE3 snapshots. mod git; +mod links; mod ownership; mod preparation; mod resolution; @@ -9,7 +10,7 @@ mod tree; use cap_fs_ext::DirExt; use cap_std::fs::Dir; pub use cyber_server::runtime::SnapshotRecovery as Recovery; -use cyber_server::runtime::{FileDiff, Snapshot}; +use cyber_server::runtime::{FileDiff, Snapshot, SnapshotRecoveryImageKind as ImageKind}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -27,12 +28,18 @@ pub(crate) struct Options { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Entry { + #[serde(default, skip_serializing_if = "is_regular")] + kind: ImageKind, object: String, length: u64, mode: u32, uid: u32, gid: u32, } +fn is_regular(kind: &ImageKind) -> bool { + *kind == ImageKind::File +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Tree { @@ -282,10 +289,14 @@ impl Store { } size = size .checked_add( - storage::open(&journal, Path::new(&file))? - .metadata() - .map_err(fail)? - .len(), + (if file.starts_with("stage_") { + links::pin_image(&journal, std::ffi::OsStr::new(&file))? + } else { + storage::open(&journal, Path::new(&file))? + }) + .metadata() + .map_err(fail)? + .len(), ) .ok_or("Snapshot storage overflow")?; } @@ -340,6 +351,7 @@ impl Store { } } Ok(Entry { + kind: ImageKind::File, object, length: bytes.len() as u64, mode: 0o644, @@ -395,6 +407,10 @@ impl Store { || tree.location != self.root || tree.root != self.root_id || tree.files.len() > ENTRY_LIMIT + || tree + .files + .values() + .any(|entry| entry.kind != ImageKind::File) || storage::digest(&serde_json::to_vec(&tree).map_err(fail)?) != digest { return Err("Snapshot belongs to a different Location or is corrupt".into()); @@ -552,6 +568,11 @@ fn gc_location( serde_json::from_slice(&storage::read(&manifests, &name, RECORD_LIMIT)?) .map_err(fail)?; if manifest.tree.version != 1 + || manifest + .tree + .files + .values() + .any(|entry| entry.kind != ImageKind::File) || storage::digest(&serde_json::to_vec(&manifest.tree).map_err(fail)?) != name { return Err("Corrupt snapshot manifest".into()); diff --git a/crates/cyber-snapshot/src/copy/ownership.rs b/crates/cyber-snapshot/src/copy/ownership.rs index 9b5af03eb..4b13b29c6 100644 --- a/crates/cyber-snapshot/src/copy/ownership.rs +++ b/crates/cyber-snapshot/src/copy/ownership.rs @@ -43,6 +43,13 @@ struct DirectoryObservation { } impl Observations { + pub(super) fn has_links(&self) -> bool { + self.files + .iter() + .flat_map(|slots| [&slots.source, &slots.stage]) + .flatten() + .any(|image| image.entry.kind == super::ImageKind::Symlink) + } pub fn validate(&self) -> Result<()> { if self.files.len() > ENTRY_LIMIT || self.parents.len() > ENTRY_LIMIT + 1 { return Err("Recovery observation limit exceeded".into()); @@ -77,6 +84,7 @@ impl Observations { identity: "0".repeat(storage::IDENTITY_MAX_BYTES), generation: "0".repeat(64), entry: Entry { + kind: super::ImageKind::File, object: "0".repeat(64), length: u64::MAX, mode: u32::MAX, @@ -85,8 +93,16 @@ impl Observations { }, }; for slots in &mut value.files { - slots.source = Some(image.clone()); - slots.stage = Some(image.clone()); + let mut bound = image.clone(); + if [&slots.source, &slots.stage] + .into_iter() + .flatten() + .any(|image| image.entry.kind == super::ImageKind::Symlink) + { + bound.entry.kind = super::ImageKind::Symlink; + } + slots.source = Some(bound.clone()); + slots.stage = Some(bound); } for parent in &mut value.parents { let bound = DirectoryObservation { @@ -100,7 +116,7 @@ impl Observations { } } -fn generation(file: &std::fs::File) -> Result { +pub(super) fn generation(file: &std::fs::File) -> Result { use std::os::unix::fs::MetadataExt; let m = file.metadata().map_err(fail)?; let value = ( @@ -132,11 +148,11 @@ pub(super) fn observe_image( Err(error) => return Err(fail(error)), Ok(_) => {} } - let pin = storage::open(parent, Path::new(name))?; + let pin = super::links::pin_image(parent, name)?; let before = generation(&pin)?; let (identity, entry) = observed(store, Some(parent), name)?.ok_or("Recovery image disappeared")?; - let current = storage::open(parent, Path::new(name))?; + let current = super::links::pin_image(parent, name)?; if storage::identity(&pin)? != identity || storage::identity(¤t)? != identity || generation(&pin)? != before @@ -294,10 +310,10 @@ fn observed( Ok(_) => {} } // Keep the first inode pinned through hashing and the second route lookup. - let pin = storage::open(dir, Path::new(name))?; + let pin = super::links::pin_image(dir, name)?; let identity = storage::identity(&pin)?; let entry = store.inspect_entry(dir, name)?; - let current = storage::open(dir, Path::new(name))?; + let current = super::links::pin_image(dir, name)?; if storage::identity(¤t)? != identity { return Err("Restore file identity changed while inspecting".into()); } @@ -320,7 +336,12 @@ fn valid_entry(entry: &Entry) -> bool { storage::valid_digest(&entry.object) && entry.length <= 64 * 1024 * 1024 && entry.mode & !0o777 == 0 - && entry.mode & 0o222 != 0 + && match entry.kind { + super::ImageKind::File => entry.mode & 0o222 != 0, + super::ImageKind::Symlink => { + entry.length > 0 && entry.length <= super::links::TARGET_LIMIT as u64 + } + } } impl File { @@ -346,13 +367,17 @@ impl File { return Err("Restore source/stage changed before ownership preparation".into()); } if let (Some((identity, _)), Some(parent)) = (&source, parent) { - let pin = storage::open(parent, Path::new(leaf))?; + let pin = super::links::pin_image(parent, leaf)?; if storage::identity(&pin)? != *identity { return Err("Restore original inode changed".into()); } // Make the actual displaced preimage durable before any source rename. // Read-only inspection deliberately never performs this preparation. - storage::sync_file(&pin)?; + if pin.metadata().map_err(fail)?.file_type().is_symlink() { + storage::sync(parent)?; + } else { + storage::sync_file(&pin)?; + } } Ok(Self { source: source.map(|(id, _)| id), diff --git a/crates/cyber-snapshot/src/copy/ownership_tests.rs b/crates/cyber-snapshot/src/copy/ownership_tests.rs index d13fe565b..260214feb 100644 --- a/crates/cyber-snapshot/src/copy/ownership_tests.rs +++ b/crates/cyber-snapshot/src/copy/ownership_tests.rs @@ -15,11 +15,19 @@ struct Prepared { impl Prepared { fn new() -> Self { + Self::with_links(false) + } + fn with_links(links: bool) -> Self { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("work"); std::fs::create_dir_all(root.join("existing")).unwrap(); - std::fs::write(root.join("existing/replace"), b"before").unwrap(); - std::fs::write(root.join("existing/delete"), b"deleted").unwrap(); + if links { + std::os::unix::fs::symlink("before", root.join("existing/replace")).unwrap(); + std::os::unix::fs::symlink("deleted", root.join("existing/delete")).unwrap(); + } else { + std::fs::write(root.join("existing/replace"), b"before").unwrap(); + std::fs::write(root.join("existing/delete"), b"deleted").unwrap(); + } let store = Store::open( &temp.path().join("data"), &root.to_string_lossy(), @@ -33,6 +41,10 @@ impl Prepared { .unwrap(); let original = store.current("existing/replace").unwrap().unwrap(); let mut after = store.object(b"after").unwrap(); + if links { + after.kind = super::super::ImageKind::Symlink; + after.mode = original.mode; + } after.uid = original.uid; after.gid = original.gid; let changes = vec![ @@ -59,13 +71,14 @@ impl Prepared { Store::record(&dir, &record).unwrap(); for (index, change) in record.changes.iter().enumerate() { if let Some(after) = &change.after { - let file = storage::create( - &dir, - &format!("stage_{index}"), - &store.bytes(after).unwrap(), - ) - .unwrap(); - Store::stage_metadata(&file, after).unwrap(); + store + .stage_image( + &dir, + &format!("stage_{index}"), + after, + &store.bytes(after).unwrap(), + ) + .unwrap(); } } record.ownership = Some(Ownership::prepare(&store, &dir, &record.changes).unwrap()); @@ -521,12 +534,19 @@ impl Drop for ChildGuard { #[test] fn process_kill_at_parent_and_file_cuts_recovers_ownership_without_progress_or_replay() { for cut in 0..5 { - verify_killed_cut(cut); + verify_killed_cut(cut, false); } } -fn verify_killed_cut(cut: usize) { - let mut p = Prepared::new(); +#[test] +fn process_kill_at_link_parent_and_exchange_cuts_preserves_typed_inode_evidence() { + for cut in 0..5 { + verify_killed_cut(cut, true); + } +} + +fn verify_killed_cut(cut: usize, links: bool) { + let mut p = Prepared::with_links(links); let base = p._temp.path().to_owned(); drop(p.store); drop(p.dir); @@ -555,7 +575,11 @@ fn verify_killed_cut(cut: usize) { ); std::thread::sleep(std::time::Duration::from_millis(10)); } - assert_live_cut(&p.root, cut); + if links { + assert_live_link_cut(&p.root, cut); + } else { + assert_live_cut(&p.root, cut); + } child.0.kill().unwrap(); child.0.wait().unwrap(); p.store = Store::open( @@ -573,6 +597,25 @@ fn verify_killed_cut(cut: usize) { assert_recovered_cut(&p, cut); } +fn assert_live_link_cut(root: &Path, cut: usize) { + assert_eq!(root.join("new/deep").exists(), cut >= 1); + assert_eq!( + std::fs::read_link(root.join("existing/replace")).unwrap(), + Path::new(if cut >= 2 { "after" } else { "before" }) + ); + assert_eq!( + std::fs::symlink_metadata(root.join("existing/delete")).is_ok(), + cut < 3 + ); + assert_eq!( + std::fs::symlink_metadata(root.join("new/deep/file")).is_ok(), + cut >= 4 + ); + for target in ["before", "after", "deleted"] { + assert!(!root.join("existing").join(target).exists()); + } +} + fn assert_live_cut(root: &Path, cut: usize) { // Independently observe the real child effect; its marker alone is not proof. assert_eq!(root.join("new/deep").exists(), cut >= 1); @@ -604,9 +647,51 @@ fn assert_recovered_cut(p: &Prepared, cut: usize) { } assert!(p.store.ensure_recovered().is_err()); if cut >= 2 { - assert_eq!(storage::read(&p.dir, "stage_0", 65536).unwrap(), b"before"); + assert_eq!(read_cut_preimage(p, "stage_0"), b"before"); } if cut >= 3 { - assert_eq!(storage::read(&p.dir, "stage_1", 65536).unwrap(), b"deleted"); + assert_eq!(read_cut_preimage(p, "stage_1"), b"deleted"); } } + +fn read_cut_preimage(p: &Prepared, name: &str) -> Vec { + if p.record.changes[0].before.as_ref().unwrap().kind == super::super::ImageKind::Symlink { + super::super::links::read(&p.dir, std::ffi::OsStr::new(name), 65536) + .unwrap() + .1 + } else { + storage::read(&p.dir, name, 65536).unwrap() + } +} + +#[test] +fn same_target_foreign_link_never_acquires_original_source_or_stage_ownership() { + for applied in [false, true] { + let p = Prepared::with_links(true); + if applied { + p.apply(0); + } + let path = p.root.join("existing/replace"); + let target = std::fs::read_link(&path).unwrap(); + std::fs::rename(&path, p.root.join("owner-retained-link")).unwrap(); + std::os::unix::fs::symlink(&target, &path).unwrap(); + assert!(p.inspect().is_err()); + assert!( + p.store + .install(&p.dir, 0, &p.record.changes[0], p.proof(), &|| Ok(())) + .is_err() + ); + assert_eq!(std::fs::read_link(&path).unwrap(), target); + assert!(p.store.ensure_recovered().is_err()); + } + let p = Prepared::with_links(true); + let stage = p.private_path("stage_0"); + let target = std::fs::read_link(&stage).unwrap(); + std::fs::rename(&stage, p.private_path("owner-retained-stage")).unwrap(); + std::os::unix::fs::symlink(&target, &stage).unwrap(); + assert!(p.inspect().is_err()); + assert_eq!( + std::fs::read_link(p.root.join("existing/replace")).unwrap(), + Path::new("before") + ); +} diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs index 347460a25..284daaba1 100644 --- a/crates/cyber-snapshot/src/copy/resolution.rs +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -117,7 +117,11 @@ impl Preimages<'_> { { return Err("Original native preimage changed before preparation".into()); } - let bytes = storage::read(&self.dir, &name, expected.length)?; + let bytes = if expected.kind == super::ImageKind::Symlink { + super::links::read(&self.dir, std::ffi::OsStr::new(&name), expected.length)?.1 + } else { + storage::read(&self.dir, &name, expected.length)? + }; if bytes.len() as u64 != expected.length || storage::digest(&bytes) != expected.object || observe_image(store, Some(&self.dir), std::ffi::OsStr::new(&name))?.as_ref() @@ -134,6 +138,7 @@ fn image(value: &ImageObservation) -> Image { } fn expected_image(entry: &super::Entry, native_identity: Option) -> Image { Image { + kind: entry.kind, hash: entry.object.clone(), bytes: entry.length, mode: entry.mode, @@ -153,7 +158,7 @@ fn resolution_id(original: &SnapshotOrigin, request: &str) -> Result { Ok(format!("restore_resolution_{}", storage::digest(&key))) } fn immutable_digest(record: &Journal) -> Result { - Ok(storage::digest( + let legacy = storage::digest( &serde_json::to_vec(&( record.version, &record.location, @@ -166,7 +171,14 @@ fn immutable_digest(record: &Journal) -> Result { &record.resolution, )) .map_err(fail)?, - )) + ); + // Existing writer receipts keep exactly their original digest framing. + match &record.backend { + None => Ok(legacy), + Some(backend) => Ok(storage::digest( + &serde_json::to_vec(&("snapshot-journal-v5", backend, legacy)).map_err(fail)?, + )), + } } fn fingerprint( @@ -723,3 +735,35 @@ fn receipt(id: &str, record: &Journal) -> Result { resolution_digest: Some(immutable_digest(record)?), }) } + +#[cfg(test)] +mod digest_tests { + use super::*; + #[test] + fn historical_file_receipts_keep_their_original_wire_digest_and_new_kinds_are_bound() { + let entry = format!( + r#"{{"object":"{}","length":1,"mode":420,"uid":1,"gid":2}}"#, + "a".repeat(64) + ); + let change = format!(r#"{{"path":"file","before":{entry},"after":null}}"#); + let encoded = format!( + r#"{{"version":3,"location":"/fixture","root":"fixture","created":0,"completed":false,"changes":[{change}],"applied":[]}}"# + ); + let mut record: Journal = serde_json::from_str(&encoded).unwrap(); + // Historical writer framing, independently pinned as JSON bytes. + let legacy_frame = format!(r#"[3,"/fixture","fixture",0,[{change}],null,null,null,null]"#); + assert_eq!( + immutable_digest(&record).unwrap(), + storage::digest(legacy_frame.as_bytes()) + ); + assert_eq!(serde_json::to_string(&record).unwrap(), encoded); + record.version = 5; + record.backend = Some(super::super::restore::JournalBackend::Copy); + let file = immutable_digest(&record).unwrap(); + record.changes[0].before.as_mut().unwrap().kind = super::super::ImageKind::Symlink; + let link = immutable_digest(&record).unwrap(); + assert_ne!(file, link); + record.backend = Some(super::super::restore::JournalBackend::Git); + assert_ne!(link, immutable_digest(&record).unwrap()); + } +} diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 1e667ea71..e70399794 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -18,12 +18,23 @@ pub(super) struct Change { pub(super) before: Option, pub(super) after: Option, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum JournalBackend { + Copy, + Git, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct Journal { // Version 3 adds prepared native file/parent ownership. Version 2 binds // only the worker and cannot acquire that historical proof retroactively. + // Version 4 binds Git context. Version 5 adds explicit backend and link kinds; + // older records retain their original file-only encoding and digest framing. pub version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backend: Option, pub location: std::path::PathBuf, pub root: String, pub created: u64, @@ -88,14 +99,40 @@ impl Store { .map_err(fail)?; if !matches!( (record.version, record.origin.is_some()), - (1, false) | (2, true) | (3, _) | (4, _) + (1, false) | (2, true) | (3, _) | (4, _) | (5, _) ) || (record.version == 4 && !record .context .as_ref() .is_some_and(|context| context.git.is_some())) + || (record.version < 5 && record.backend.is_some()) + || (record.version == 5 + && match &record.backend { + Some(JournalBackend::Git) => !record + .context + .as_ref() + .is_some_and(|context| context.git.is_some()), + Some(JournalBackend::Copy) => record + .context + .as_ref() + .is_some_and(|context| context.git.is_some()), + None => true, + }) || record.location != self.root || record.root != self.root_id + || (record.version < 5 + && (record.changes.iter().any(|change| { + [&change.before, &change.after] + .into_iter() + .flatten() + .any(|entry| entry.kind == super::ImageKind::Symlink) + }) || record.resolution.as_ref().is_some_and(|resolution| { + resolution.review.has_links() + || resolution + .outcome + .as_ref() + .is_some_and(|outcome| outcome.has_links()) + }))) || record.changes.len() > ENTRY_LIMIT || record.applied.len() > record.changes.len() || !record @@ -281,6 +318,12 @@ impl Store { let (Some(current), Some(base), Some(want)) = (current, base, want) else { return Ok(None); }; + if [current, base, want] + .iter() + .any(|entry| entry.kind != super::ImageKind::File) + { + return Ok(None); + } let (ours, base, theirs) = (self.bytes(current)?, self.bytes(base)?, self.bytes(want)?); let (Ok(ours), Ok(base), Ok(theirs)) = ( std::str::from_utf8(&ours), @@ -391,9 +434,17 @@ impl Store { None => self.bytes(after), } .map_err(RestoreError::Failed)?; - let file = storage::create(&dir, &format!("stage_{i}"), &bytes) + if after.kind == super::ImageKind::Symlink && journal.resolution.is_none() { + let boundary = journal + .context + .as_ref() + .and_then(|context| context.git.as_ref()) + .map_or(self.root.as_path(), |git| git.worktree.as_path()); + self.check_link_target(&change.path, &bytes, boundary) + .map_err(RestoreError::Failed)?; + } + self.stage_image(&dir, &format!("stage_{i}"), after, &bytes) .map_err(RestoreError::Failed)?; - Self::stage_metadata(&file, after).map_err(RestoreError::Failed)?; } } storage::sync(&dir).map_err(RestoreError::Failed)?; @@ -416,6 +467,21 @@ impl Store { Ok(()) }; verify()?; + if journal.resolution.is_none() + && let Some(after) = &journal.changes[i].after + && after.kind == super::ImageKind::Symlink + { + let boundary = journal + .context + .as_ref() + .and_then(|context| context.git.as_ref()) + .map_or(self.root.as_path(), |git| git.worktree.as_path()); + self.check_link_target( + &journal.changes[i].path, + &self.bytes(after)?, + boundary, + )?; + } self.install( &dir, i, @@ -489,13 +555,32 @@ impl Store { let version = if context .as_ref() .is_some_and(|context| context.git.is_some()) + || changes.iter().any(|change| { + [&change.before, &change.after] + .into_iter() + .flatten() + .any(|entry| entry.kind == super::ImageKind::Symlink) + }) + || resolution + .as_ref() + .is_some_and(|resolution| resolution.review.has_links()) { - 4 + 5 } else { 3 }; let mut journal = Journal { version, + backend: (version == 5).then(|| { + if context + .as_ref() + .is_some_and(|context| context.git.is_some()) + { + JournalBackend::Git + } else { + JournalBackend::Copy + } + }), location: self.root.clone(), root: self.root_id.clone(), created: now(), @@ -582,6 +667,20 @@ impl Store { { return Err("Restore filesystem completion differs from native ownership".into()); } + if record.resolution.is_none() { + let boundary = record + .context + .as_ref() + .and_then(|context| context.git.as_ref()) + .map_or(self.root.as_path(), |git| git.worktree.as_path()); + for change in &record.changes { + if let Some(after) = &change.after + && after.kind == super::ImageKind::Symlink + { + self.check_link_target(&change.path, &self.bytes(after)?, boundary)?; + } + } + } Ok(()) } @@ -641,7 +740,7 @@ impl Store { journal.acknowledged = true; Self::record(&dir, &journal) } - fn stage_metadata(file: &std::fs::File, entry: &Entry) -> Result<()> { + pub(super) fn stage_metadata(file: &std::fs::File, entry: &Entry) -> Result<()> { rustix::fs::fchown( file, Some(rustix::fs::Uid::from_raw(entry.uid)), @@ -681,6 +780,27 @@ impl Store { .ok_or("Invalid restore leaf")?; let stage = format!("stage_{index}"); match (&change.before, &change.after) { + (Some(before), Some(_)) if before.kind == super::ImageKind::Symlink => { + let original = super::links::pin_image(&parent, leaf)?; + ownership.verify_original(index, &original)?; + if ownership.inspect_change(self, dir, index, change)? != Effect::Unapplied { + return Err("Symbolic-link source changed before exchange".into()); + } + self.authorize_restore_effect(dir, ownership, &change.path, &parent_id, authorize)?; + rustix::fs::renameat_with( + dir, + &stage, + &parent, + leaf, + rustix::fs::RenameFlags::EXCHANGE, + ) + .map_err(fail)?; + if self.inspect_entry(dir, std::ffi::OsStr::new(&stage))? != *before { + return Err( + "Displaced symbolic-link preimage changed; recovery required".into(), + ); + } + } (Some(before), Some(_)) => { let target = cyber_core::binary::native::Target::capture( &self.root, @@ -801,6 +921,7 @@ mod tests { path, before: None, after: Some(Entry { + kind: crate::copy::ImageKind::File, object: blake3::hash(b"").to_hex().to_string(), length: 0, mode: 0o644, @@ -866,6 +987,7 @@ mod tests { Store::record( &dir, &Journal { + backend: None, version: 2, location: store.root.clone(), root: store.root_id.clone(), diff --git a/crates/cyber-snapshot/src/copy/tree.rs b/crates/cyber-snapshot/src/copy/tree.rs index 23a6e5282..1862da812 100644 --- a/crates/cyber-snapshot/src/copy/tree.rs +++ b/crates/cyber-snapshot/src/copy/tree.rs @@ -140,6 +140,14 @@ impl Store { self.entry(dir, name, false) } fn entry(&self, dir: &Dir, name: &std::ffi::OsStr, save: bool) -> Result { + if dir + .symlink_metadata(name) + .map_err(fail)? + .file_type() + .is_symlink() + { + return self.link_entry(dir, name, save); + } let mut file = storage::open(dir, Path::new(name))?; let before = file.metadata().map_err(fail)?; let metadata = cyber_core::binary::native::plain_snapshot_metadata(&file) @@ -171,6 +179,7 @@ impl Store { self.object(&bytes)? } else { Entry { + kind: super::ImageKind::File, object: storage::digest(&bytes), length: bytes.len() as u64, mode: 0, diff --git a/docs/rewind.md b/docs/rewind.md index 9b783a685..f87e358cd 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -331,3 +331,12 @@ Git restore resolves file paths literally and verifies each blob image against i Git object ID. Corrupt blob data or ambiguous image listings refuse before source replacement. Retain the snapshot data for diagnosis; this check does not provide the still-unfinished owned Git crash-recovery workflow. + + +Recovery image views distinguish `kind: "file"` from `kind: "symlink"`; a missing +kind means file, including in legacy responses. A link image's hash and byte length describe +the link target text, not the contents of its target file. Native recovery never +follows that target while staging, exchanging or reading retained preimages. +The local Git journal bridge supports typed native link fixtures, but ordinary +owned Git recovery remains unavailable until its remaining lifecycle, full-state +and platform acceptance gates are complete. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index fd608f2c2..edd8a446f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1799,3 +1799,87 @@ pinned OpenSpec passed all 117 items; spec lint reports 0 errors/25 warnings. Fu workspace tests, SDK execution, real CLI/TUI/App acceptance, native process-kill Git acceptance and Linux/Windows execution were not run for this increment. The backend fixture keep/rollback tests are not runtime lifecycle or crash evidence. + + +## Native symbolic-link images and retained preimages (2026-11-07) + +`copy/links.rs` now pins symbolic-link inodes directly: macOS uses O_SYMLINK and +Linux uses O_PATH/O_NOFOLLOW. It checks unique inode identity, generation, bounded +target bytes (4,096 bytes), ownership/mode and supported metadata without following +the target. macOS reuses a new additive plain-link metadata helper in core; existing +binary regular-file behavior is unchanged. Linux's O_PATH descriptor cannot read +xattrs directly, so metadata-only llistxattr uses the kernel proc descriptor route +for the pinned parent, checks its identity, and checks parent/link generations +around inspection. Missing proc routing or unsupported metadata refuses. Linux +execution of this path remains unverified locally. + +Native images distinguish file content from link target bytes. Existing file +entries omit the new field, and legacy journals omit the backend field; their +original serialization and immutable digest framing remain intact. Version-5 +intents bind an explicit copy/Git backend and typed images in a new digest frame. +Version-4 Git contexts remain readable with their original frame. Older journals +cannot acquire link ownership merely by adding a kind field. Non-Git capture +continues to report links as unsupported and legacy copy manifests remain file-only. + +The existing journal executor stages typed link entries, preserves displaced +inodes, verifies prepared source/stage identity, exchanges without following links +and revalidates the outcome. Recovery reads exact retained target bytes under +native observations, stages independent rollback entries and preserves unrelated +target file contents. Three-way text merging is restricted to regular files. New +Git-desired links are checked before staging, before publication and at completion/ +consumption against the worktree and protected/private roots. Resolution rollback +may reinstate an exact owned preimage without following its target. + +Review found and fixed an escape when lexical normalization cancelled an alias +before `..`; physical checks now resolve the original component order. Positives +cover link-to-link, file-to-link, link-to-file, create/delete and safe dangling +targets with rollback. Negatives cover escaping/protected aliases, same-target +foreign source and stage inodes, hardlinked links, bounds and link xattrs. Five +actual child-process kills at parent/replace/delete/create cuts reopen the private +store under a new owner, classify real typed inode slots without progress records, +retain preimages and keep the recovery fence. These native fixtures do not confer +runtime origin/acknowledgement authority and do not prove all Git service cuts. + +Owner API images now carry `kind: file | symlink`; omitted legacy fields default +to file. OpenAPI and the TypeScript SDK are regenerated. The code-review skill +covered all changed core/adapter/codec/API/test/artifact files, including no-follow +semantics, old digest compatibility, exact private slots, quota accounting and +source-target distinction. The additive core helper does not change binary editor +contracts. No critical/major issue remains in this disabled bridge scope. + +The Git availability/reservation gates remain disabled. Complete expected-state +scanning, file/directory transitions, literal backslash paths in the copy engine, +complex link-target compatibility, actual Git worker lifetime/receipt integration, +reference-aware reclamation, full clients and native platform/service-kill matrix +remain required. No parent task is accepted by this increment. + +Verification on local macOS arm64: the final snapshot suite passed 113 cases with +two ignored fixtures (the pathspec fixture is invoked by its parent), including the +five real link kill/reopen cuts. The server unit suite passed 429 cases with one +ignored fixture; the final file-wire compatibility refinement also passed its +focused server regression. All five core binary-native regressions passed. +All-target Clippy for snapshot/server/core passed with warnings denied. OpenAPI +export and current-source consistency passed after refreshing the schema annotation +changed by omitting ordinary file kinds. SDK typecheck passed; SDK tests passed +134 cases with 10 configured real-server skips because CYBER_BIN was unset. The +TypeScript output stayed unchanged through that schema-annotation refresh. +Formatting, generated SDK and diff checks passed; strict pinned OpenSpec passed all +117 items and spec lint reports 0 errors/25 warnings. An initial test-only ImageKind +namespace compile error was corrected before the passing full runs. The ordinary +file-image API encoding remains byte-compatible by omitting its default kind, +while symlink images serialize the explicit kind. + +Full-workspace tests, real CLI/TUI/App Git recovery, native Linux/Windows execution +and the full service/writer/acknowledgement/reclamation crash matrix were not run +for this increment. The Linux proc metadata path is implemented but not claimed +verified. Generic file-copy capture still reports links as unsupported. The complete +Git recovery workflow and parent acceptance tasks remain open. + +Additional target-node checks compare protected reserved siblings and configured +private ancestors by device/inode identity. Tests cover actual case aliases when +the filesystem supports them, including a direct node-check test independent of +canonical spelling. The native macOS resolver already normalizes the tested casing; +an initial Python-only probe was not equivalent and is not claimed as a native +resolver defect. One test initially expected only the identity-check error text; it +was corrected to accept the earlier protected-path refusal as well. The final full +snapshot run and three-package all-target Clippy pass include these checks. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 66826041e..d7e139935 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -481,3 +481,19 @@ version binds Git identity and Location. Legacy copy contexts keep their existin encoded identity. Git admission remains disabled until symbolic links, full expected state revalidation and all native recovery/acknowledgement paths are integrated; regular-file import tests alone do not authorize that switch. + + +### Native symbolic-link images + +Native Git journals distinguish regular file bytes from symbolic-link target bytes. +Link observation pins the link inode itself without following its target, binds +identity/generation, owner/mode and bounded target bytes, and refuses hardlinked or +unsupported metadata. Source and private slots use the same typed observations; +regular-file legacy encodings retain their original identity. New link-capable +intents are versioned separately and cannot be read as legacy file-only proof. +Staging and exchange operate on the link namespace entry, never the target file. +Recovery review identifies the image kind. Rollback reads exact retained target +bytes under the original native observation before independent intent preparation. +New Git-desired links must stay inside the worktree and outside protected/private +roots; rollback may reinstate an exact owned preimage without following it. This +does not authorize broad Git admission before full state/lifetime/acceptance gates. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 2cdc052ba..02d4f1cd5 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -144,6 +144,14 @@ - **AND** native source changes SHALL retain displaced preimages and use the same ownership, recovery review and retry-lookup rules as file-copy journals - **AND** legacy copy contexts SHALL preserve their encoded identity and SHALL NOT be reinterpreted as Git contexts +#### Scenario: Native symbolic-link recovery images +- **WHEN** a Git symbolic-link image enters a native restore or owner recovery plan +- **THEN** the implementation SHALL pin the link inode without following its target and bind typed target bytes, native identity/generation and supported metadata +- **AND** native staging, replacement and retained-preimage reads SHALL operate on the link namespace entry without reading or writing target file data +- **AND** new desired links SHALL refuse escaping or protected targets, including escapes through an existing alias followed by `..` +- **AND** review SHALL identify file versus symbolic-link images, while legacy images default to files +- **AND** link-capable journals SHALL use a distinct version/backend binding without promoting older file-only ownership or changing historical file receipt digests + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 67111719b..6f720d1f8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -280,3 +280,15 @@ Git worker reservation/admission is still disabled. Symbolic links, full expecte state scanning (including newly appearing paths), exact runtime Git lifetime/ receipt acknowledgement, retention and complete crash/client/platform acceptance remain required; M2–M8 and parent tasks remain open. + + +Native link continuation: version-5 intents now bind backend and typed regular/link +images while preserving earlier file encodings and digest frames. Link observations +pin the link inode, reject aliases/unsupported metadata and read bounded target +bytes without following them. Shared native staging/exchange and exact-preimage +rollback support link/file transitions. Review and generated SDK expose image kind. +Local macOS native positives and five real link process-kill cuts cover inode +classification without progress/replay; these are backend fixtures, not complete +Git runtime acceptance. Ordinary Git admission remains disabled for full expected +state/Git compatibility, lifetime/acknowledgement, retention, clients and complete +crash/platform gates. M2–M8 and parent tasks remain open. diff --git a/sdk/openapi.json b/sdk/openapi.json index cec042254..6a1c30b0f 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7314,6 +7314,9 @@ "SnapshotRecoveryImage": { "type": "object", "properties": { + "kind": { + "$ref": "#/components/schemas/SnapshotRecoveryImageKind" + }, "hash": { "type": "string" }, @@ -7353,6 +7356,13 @@ "gid" ] }, + "SnapshotRecoveryImageKind": { + "type": "string", + "enum": [ + "file", + "symlink" + ] + }, "SnapshotRecoveryConfirmation": { "type": "object", "additionalProperties": false, diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index ce0e2daaf..5fae75ad4 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -4800,6 +4800,7 @@ export type SnapshotRecoveryConfirmation = { }; export type SnapshotRecoveryImage = { + kind?: SnapshotRecoveryImageKind; hash: string; bytes: number; mode: number; @@ -4809,6 +4810,8 @@ export type SnapshotRecoveryImage = { native_identity?: string | null; }; +export type SnapshotRecoveryImageKind = "file" | "symlink"; + export type SnapshotRecoveryPath = { path: string; /** `applied`, `unapplied`, or `foreign`; foreign files are never rollback targets. */ From 878bd56428b5338503e30eee8f17f47a2ac5b8f4 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 03:31:54 -0400 Subject: [PATCH 23/87] Validate native Git namespace and rule inputs before journaling --- crates/cyber-snapshot/src/copy/git.rs | 2 + crates/cyber-snapshot/src/copy/git/tests.rs | 299 ++++++++++++++++++ crates/cyber-snapshot/src/copy/git_rules.rs | 205 ++++++++++++ crates/cyber-snapshot/src/copy/inventory.rs | 191 +++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 2 + crates/cyber-snapshot/src/copy/tree.rs | 4 +- crates/cyber-snapshot/src/git/prepared.rs | 70 +++- .../implementation.md | 68 ++++ .../managed-recovery.md | 29 ++ .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 11 + 11 files changed, 885 insertions(+), 3 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/git_rules.rs create mode 100644 crates/cyber-snapshot/src/copy/inventory.rs diff --git a/crates/cyber-snapshot/src/copy/git.rs b/crates/cyber-snapshot/src/copy/git.rs index 9a98ffeca..1b1571b6c 100644 --- a/crates/cyber-snapshot/src/copy/git.rs +++ b/crates/cyber-snapshot/src/copy/git.rs @@ -154,6 +154,7 @@ impl Store { "Git journal path count exceeds limit".into(), )); } + let inventory = self.git_inventory(prepared)?; let mut changes = Vec::new(); let mut conflicts = Vec::new(); for path in paths { @@ -215,6 +216,7 @@ impl Store { if !conflicts.is_empty() { return Err(RestoreError::Conflict(conflicts)); } + inventory.verify(self)?; Ok((changes, context)) } diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index c61bd721b..6b034443c 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -627,3 +627,302 @@ async fn native_link_targets_cannot_use_case_aliases_of_configured_private_roots assert_node(&repo.root, "file"); assert!(store.recovery().unwrap().is_empty()); } + +#[tokio::test] +async fn native_inventory_refuses_new_files_nested_files_and_links_before_journaling() { + for kind in ["file", "nested", "link"] { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + match kind { + "file" => std::fs::write(repo.root.join("appeared"), b"new owner").unwrap(), + "nested" => { + std::fs::create_dir(repo.root.join("appeared")).unwrap(); + std::fs::write(repo.root.join("appeared/child"), b"new owner").unwrap(); + } + _ => std::os::unix::fs::symlink("../never-opened", repo.root.join("appeared")).unwrap(), + } + let store = repo.store(&repo.root); + assert!( + matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + ), + "{kind}" + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} + +#[tokio::test] +async fn native_inventory_respects_live_git_rules_in_mixed_directories_and_empty_directories() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitignore"), b"*.log\nignored/\n").unwrap(); + std::fs::create_dir(repo.root.join("mixed")).unwrap(); + std::fs::create_dir(repo.root.join("ignored")).unwrap(); + std::fs::write(repo.root.join("mixed/owner.log"), b"ignored owner").unwrap(); + std::fs::write(repo.root.join("mixed/kept"), b"kept owner").unwrap(); + std::fs::write(repo.root.join("ignored/seed"), b"ignored seed").unwrap(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + std::fs::write(repo.root.join("mixed/owner.log"), b"later ignored owner").unwrap(); + std::fs::write(repo.root.join("ignored/new"), b"later ignored owner").unwrap(); + std::fs::create_dir_all(repo.root.join("empty/deep")).unwrap(); + let store = repo.store(&repo.root); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); + assert_eq!( + std::fs::read(repo.root.join("mixed/owner.log")).unwrap(), + b"later ignored owner" + ); + assert_eq!( + std::fs::read(repo.root.join("mixed/kept")).unwrap(), + b"kept owner" + ); + assert!(repo.root.join("empty/deep").is_dir()); +} + +#[tokio::test] +async fn namespace_and_rule_changes_after_inventory_are_stale_without_source_effects() { + for change in ["namespace", "rule"] { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitignore"), b"*.log\n").unwrap(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + let inventory = store.git_inventory(&prepared).unwrap(); + if change == "namespace" { + std::fs::write(repo.root.join("appeared"), b"new owner").unwrap(); + } else { + std::fs::write(repo.root.join(".gitignore"), b"file\n").unwrap(); + } + assert!( + matches!(inventory.verify(&store), Err(RestoreError::Stale)), + "{change}" + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} + +#[tokio::test] +async fn historical_exclusions_protect_mutation_without_hiding_current_state_changes() { + for owner_changes in [false, true] { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitignore"), b"kept\n").unwrap(); + std::fs::write(repo.root.join("kept"), b"owner before").unwrap(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join(".gitignore"), b"").unwrap(); + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + if owner_changes { + std::fs::write(repo.root.join("kept"), b"later owner").unwrap(); + } + let store = repo.store(&repo.root); + let result = store.restore_git_worker(prepared, None); + if owner_changes { + assert!(matches!(result, Err(RestoreError::Stale))); + assert!(store.recovery().unwrap().is_empty()); + } else { + assert!(result.is_ok(), "{result:?}"); + } + assert_eq!( + std::fs::read(repo.root.join("kept")).unwrap(), + if owner_changes { + b"later owner".as_slice() + } else { + b"owner before".as_slice() + } + ); + } +} + +#[tokio::test] +async fn native_inventory_keeps_literal_backslashes_in_locations_and_file_names() { + let repo = Repo::new(); + let directory = repo.root.join("back\\slash"); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("literal\\file"), b"before").unwrap(); + let target = repo.track(&directory).await; + std::fs::write(directory.join("literal\\file"), b"after").unwrap(); + let recorded = repo.track(&directory).await; + let prepared = repo + .prepare(&directory, &target, &recorded, &recorded) + .await; + let store = repo.store(&directory); + assert_eq!( + store.restore_git_worker(prepared, None).unwrap(), + ["literal\\file"] + ); + assert_eq!( + std::fs::read(directory.join("literal\\file")).unwrap(), + b"before" + ); + for invalid in ["../escape", "a//b", "a/./b", "/absolute"] { + assert!(super::super::tree::safe_path(invalid).is_err()); + } +} + +#[tokio::test] +async fn native_inventory_applies_ancestor_rules_and_nested_negations_only_inside_its_location() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitignore"), b"*.tmp\n").unwrap(); + let directory = repo.root.join("scope"); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join(".gitignore"), b"!keep.tmp\n").unwrap(); + std::fs::write(directory.join("keep.tmp"), b"included owner").unwrap(); + std::fs::write(directory.join("owner.tmp"), b"ignored owner").unwrap(); + std::fs::write(directory.join("file"), b"before").unwrap(); + let target = repo.track(&directory).await; + std::fs::write(directory.join("file"), b"after").unwrap(); + let recorded = repo.track(&directory).await; + let prepared = repo + .prepare(&directory, &target, &recorded, &recorded) + .await; + std::fs::write(repo.root.join("sibling"), b"outside Location owner").unwrap(); + std::fs::write(directory.join("owner.tmp"), b"later ignored owner").unwrap(); + let store = repo.store(&directory); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); + assert_eq!( + std::fs::read(directory.join("keep.tmp")).unwrap(), + b"included owner" + ); + assert_eq!( + std::fs::read(repo.root.join("sibling")).unwrap(), + b"outside Location owner" + ); +} + +#[tokio::test] +async fn native_inventory_reads_git_external_ignore_inputs_and_detects_rule_only_changes() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + repo.track(&repo.root).await; + let global = repo.root.parent().unwrap().join("global-ignore"); + std::fs::write(&global, b"*.global\n").unwrap(); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args(["config", "core.excludesfile"]) + .arg(&global) + .status() + .unwrap() + .success() + ); + std::fs::write(repo.root.join(".git/info/exclude"), b"*.info\n").unwrap(); + std::fs::write(repo.root.join("owner.global"), b"global ignored owner").unwrap(); + std::fs::write(repo.root.join("owner.info"), b"info ignored owner").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + let inventory = store.git_inventory(&prepared).unwrap(); + std::fs::write(&global, b"file\n").unwrap(); + assert!(matches!(inventory.verify(&store), Err(RestoreError::Stale))); + std::fs::write(&global, b"*.global\n").unwrap(); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); + assert_eq!( + std::fs::read(repo.root.join("owner.info")).unwrap(), + b"info ignored owner" + ); + assert_eq!( + std::fs::read(repo.root.join("owner.global")).unwrap(), + b"global ignored owner" + ); +} + +#[tokio::test] +async fn expected_exclusion_disappearance_and_type_changes_are_stale() { + for change in ["missing", "directory", "file"] { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitignore"), b"ignored/\n*.log\n").unwrap(); + std::fs::create_dir(repo.root.join("ignored")).unwrap(); + std::fs::write(repo.root.join("ignored/seed"), b"ignored owner").unwrap(); + std::fs::write(repo.root.join("old.log"), b"ignored owner").unwrap(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + match change { + "missing" => std::fs::remove_file(repo.root.join("old.log")).unwrap(), + "directory" => { + std::fs::remove_file(repo.root.join("old.log")).unwrap(); + std::fs::create_dir(repo.root.join("old.log")).unwrap(); + } + _ => { + std::fs::remove_file(repo.root.join("ignored/seed")).unwrap(); + std::fs::remove_dir(repo.root.join("ignored")).unwrap(); + std::fs::write(repo.root.join("ignored"), b"owner file").unwrap(); + } + } + let store = repo.store(&repo.root); + assert!( + matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + ), + "{change}" + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} + +#[tokio::test] +async fn non_git_snapshots_capture_and_restore_literal_backslash_names() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("work"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("literal\\name"), b"before").unwrap(); + let snapshots = + crate::GitSnapshots::new(base.join("data"), Arc::new(|_| serde_json::json!({}))); + let directory = root.to_str().unwrap(); + let target = snapshots.track(directory).await.unwrap().unwrap().tree; + std::fs::write(root.join("literal\\name"), b"after").unwrap(); + let recorded = snapshots.track(directory).await.unwrap().unwrap().tree; + assert_eq!( + snapshots + .restore_reviewed( + directory, + &target, + &recorded, + &recorded, + Arc::new(|| Ok(())) + ) + .await + .unwrap(), + ["literal\\name"] + ); + assert_eq!( + std::fs::read(root.join("literal\\name")).unwrap(), + b"before" + ); +} diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs new file mode 100644 index 000000000..a5ce09690 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -0,0 +1,205 @@ +//! Read-only Git rule inputs. These bindings grant no mutation or recovery authority. +use super::storage::fail; +use crate::git::prepared::PreparedRestore; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions, OpenOptionsExt}; +use cyber_server::runtime::RestoreError; +use ignore::gitignore::{Gitignore, GitignoreBuilder}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +const INPUT_LIMIT: u64 = 1024 * 1024; +const TOTAL_LIMIT: usize = 8 * 1024 * 1024; +struct Binding { + path: PathBuf, + source: bool, + hash: Option, +} +pub(super) struct Rules { + root: Dir, + path: PathBuf, + case_insensitive: bool, + bindings: Vec, + stack: Vec, + bytes: usize, +} +impl Rules { + pub(super) fn new(prepared: &PreparedRestore) -> Result { + let mut rules = Self { + root: Dir::open_ambient_dir(&prepared.worktree, cap_std::ambient_authority()) + .map_err(failed)?, + path: prepared.worktree.clone(), + case_insensitive: prepared.ignore.case_insensitive, + bindings: vec![], + stack: vec![], + bytes: 0, + }; + let mut builder = rules.builder()?; + for path in prepared + .ignore + .global + .iter() + .chain(std::iter::once(&prepared.ignore.info)) + { + if let Some(bytes) = rules.bind(path, false)? { + rules.lines(&mut builder, None, &bytes)?; + } + } + rules.stack.push( + builder + .build() + .map_err(|_| failed("Unsupported Git ignore rules"))?, + ); + if !prepared.prefix.is_empty() { + rules.enter("")?; + let mut parent = Path::new(&prepared.prefix).parent(); + let mut ancestors = Vec::new(); + while let Some(path) = parent.filter(|path| !path.as_os_str().is_empty()) { + ancestors.push(path.to_path_buf()); + parent = path.parent(); + } + for path in ancestors.into_iter().rev() { + rules.enter( + path.to_str() + .ok_or_else(|| failed("Invalid Git rule scope"))?, + )?; + } + } + Ok(rules) + } + fn builder(&self) -> Result { + let mut builder = GitignoreBuilder::new(&self.path); + builder + .case_insensitive(self.case_insensitive) + .map_err(|_| failed("Unsupported Git ignore case policy"))?; + Ok(builder) + } + fn lines( + &self, + builder: &mut GitignoreBuilder, + from: Option<&Path>, + bytes: &[u8], + ) -> Result<(), RestoreError> { + let text = std::str::from_utf8(bytes) + .map_err(|_| failed("Non-UTF-8 Git rules are unsupported"))?; + for line in text.lines() { + builder + .add_line(from.map(Path::to_path_buf), line) + .map_err(|_| failed("Unsupported Git ignore pattern"))?; + } + Ok(()) + } + pub(super) fn enter(&mut self, relative: &str) -> Result<(), RestoreError> { + let path = Path::new(relative).join(".gitignore"); + let mut builder = self.builder()?; + if let Some(bytes) = self.bind(&path, true)? { + self.lines(&mut builder, Some(&self.path.join(&path)), &bytes)?; + } + self.stack.push( + builder + .build() + .map_err(|_| failed("Unsupported Git ignore rules"))?, + ); + Ok(()) + } + pub(super) fn leave(&mut self) { + self.stack.pop(); + } + pub(super) fn ignored(&self, path: &str, directory: bool) -> bool { + self.stack + .iter() + .rev() + .map(|rules| rules.matched(path, directory)) + .find(|matched| !matched.is_none()) + .is_some_and(|matched| matched.is_ignore()) + } + fn bind(&mut self, path: &Path, source: bool) -> Result>, RestoreError> { + let bytes = read(&self.root, path, source)?; + self.bytes = self + .bytes + .checked_add(path.as_os_str().len() + bytes.as_ref().map_or(0, Vec::len)) + .ok_or_else(|| failed("Git rule input size overflow"))?; + if self.bytes > TOTAL_LIMIT || self.bindings.len() >= 65536 { + return Err(failed("Git rule inputs exceed limit")); + } + self.bindings.push(Binding { + path: path.into(), + source, + hash: bytes + .as_ref() + .map(|bytes| blake3::hash(bytes).to_hex().to_string()), + }); + Ok(bytes) + } + pub(super) fn verify(&self) -> Result<(), RestoreError> { + for binding in &self.bindings { + let bytes = read(&self.root, &binding.path, binding.source)?; + if bytes + .as_ref() + .map(|bytes| blake3::hash(bytes).to_hex().to_string()) + != binding.hash + { + return Err(RestoreError::Stale); + } + } + Ok(()) + } +} +fn read(root: &Dir, path: &Path, source: bool) -> Result>, RestoreError> { + let result = if source { + let mut directory = root.try_clone().map_err(failed)?; + for part in path + .parent() + .ok_or_else(|| failed("Invalid Git rule path"))? + .components() + { + directory = directory + .open_dir_nofollow(part.as_os_str()) + .map_err(|_| RestoreError::Stale)?; + } + let name = path + .file_name() + .ok_or_else(|| failed("Invalid Git rule file"))?; + match directory.symlink_metadata(name) { + Ok(metadata) if metadata.file_type().is_symlink() => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(failed(error)), + _ => {} + } + let mut options = OpenOptions::new(); + options + .read(true) + .follow(FollowSymlinks::No) + .custom_flags(libc::O_NONBLOCK); + directory + .open_with(name, &options) + .map(|file| file.into_std()) + } else { + use std::os::unix::fs::OpenOptionsExt; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(path) + }; + let mut file = match result { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(failed(error)), + }; + let metadata = file.metadata().map_err(failed)?; + if !metadata.is_file() || metadata.len() > INPUT_LIMIT { + return Err(failed("Unsupported or over-limit Git rule input")); + } + let mut bytes = Vec::new(); + file.by_ref() + .take(INPUT_LIMIT + 1) + .read_to_end(&mut bytes) + .map_err(failed)?; + if bytes.len() as u64 > INPUT_LIMIT { + return Err(failed("Git rule input grew beyond limit")); + } + Ok(Some(bytes)) +} +fn failed(error: impl std::fmt::Display) -> RestoreError { + RestoreError::Failed(fail(error)) +} diff --git a/crates/cyber-snapshot/src/copy/inventory.rs b/crates/cyber-snapshot/src/copy/inventory.rs new file mode 100644 index 000000000..69965f709 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/inventory.rs @@ -0,0 +1,191 @@ +//! Read-only pre-effect namespace validation, distinct from native journal ownership. +use super::Store; +use super::storage::fail; +use crate::git::prepared::PreparedRestore; +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; +use cyber_server::runtime::RestoreError; +use std::collections::BTreeSet; +use std::path::Path; + +const NODE_LIMIT: usize = 65536; +const BYTE_LIMIT: usize = 8 * 1024 * 1024; + +struct DirectoryMark { + path: String, + generation: String, +} +pub(super) struct Inventory { + directories: Vec, + nodes: usize, + bytes: usize, + found: BTreeSet, + rules: super::git_rules::Rules, +} + +impl Inventory { + /// These marks detect planning races. They never authorize source mutation, + /// acknowledgement, rollback, or removal of a directory. + pub(super) fn verify(&self, store: &Store) -> Result<(), RestoreError> { + store.verify_root().map_err(RestoreError::Failed)?; + for mark in &self.directories { + let mut dir = store.work.try_clone().map_err(failed)?; + for part in Path::new(&mark.path).components() { + dir = dir + .open_dir_nofollow(part.as_os_str()) + .map_err(|_| RestoreError::Stale)?; + } + if generation(&dir)? != mark.generation { + return Err(RestoreError::Stale); + } + } + self.rules.verify()?; + store.verify_root().map_err(RestoreError::Failed) + } + + fn charge(&mut self, bytes: usize) -> Result<(), RestoreError> { + self.nodes += 1; + self.bytes = self + .bytes + .checked_add(bytes) + .ok_or_else(|| failed("Git inventory size overflow"))?; + if self.nodes > NODE_LIMIT || self.bytes > BYTE_LIMIT { + return Err(failed("Git namespace inventory exceeds limit")); + } + Ok(()) + } + + fn walk( + &mut self, + store: &Store, + prepared: &PreparedRestore, + dir: &Dir, + prefix: &str, + depth: usize, + ) -> Result<(), RestoreError> { + if depth > 64 { + return Err(failed("Git inventory directory depth exceeds limit")); + } + let rule_scope = if prepared.prefix.is_empty() { + prefix.to_owned() + } else if prefix.is_empty() { + prepared.prefix.clone() + } else { + format!("{}/{prefix}", prepared.prefix) + }; + self.rules.enter(&rule_scope)?; + let before = generation(dir)?; + self.charge(prefix.len() + before.len())?; + for entry in dir.entries().map_err(failed)? { + let entry = entry.map_err(failed)?; + let leaf = entry.file_name(); + let name = leaf + .to_str() + .ok_or_else(|| failed("Non-UTF-8 Git inventory path is unsupported"))?; + let path = if prefix.is_empty() { + name.to_owned() + } else { + format!("{prefix}/{name}") + }; + self.charge(path.len())?; + let full = if prepared.prefix.is_empty() { + path.clone() + } else { + format!("{}/{path}", prepared.prefix) + }; + super::tree::safe_path(&path).map_err(RestoreError::Failed)?; + // Syntactic validation is separate so only protected paths are skipped. + if store.check_source_path(&path).is_err() { + continue; + } + let metadata = dir.symlink_metadata(&leaf).map_err(failed)?; + if self.rules.ignored( + &full, + metadata.is_dir() && !metadata.file_type().is_symlink(), + ) { + continue; + } + if metadata.is_dir() && !metadata.file_type().is_symlink() { + let child = dir + .open_dir_nofollow(&leaf) + .map_err(|_| RestoreError::Stale)?; + self.walk(store, prepared, &child, &path, depth + 1)?; + } else { + // Unknown leaf kinds are stale without opening them. In particular, + // a new FIFO, socket or directory link is never read or traversed. + if !prepared.expected.images.contains_key(&path) { + return Err(RestoreError::Stale); + } + self.found.insert(path); + } + } + if generation(dir)? != before { + return Err(RestoreError::Stale); + } + self.rules.leave(); + self.directories.push(DirectoryMark { + path: prefix.into(), + generation: before, + }); + Ok(()) + } +} + +impl Store { + pub(super) fn git_inventory( + &self, + prepared: &PreparedRestore, + ) -> Result { + self.verify_root().map_err(RestoreError::Failed)?; + self.check_expected_exclusions(prepared)?; + let mut inventory = Inventory { + directories: vec![], + nodes: 0, + bytes: 0, + found: BTreeSet::new(), + rules: super::git_rules::Rules::new(prepared)?, + }; + inventory.walk(self, prepared, &self.work, "", 0)?; + if !inventory.found.iter().eq(prepared.expected.images.keys()) { + return Err(RestoreError::Stale); + } + inventory.verify(self)?; + Ok(inventory) + } + fn check_expected_exclusions(&self, prepared: &PreparedRestore) -> Result<(), RestoreError> { + for excluded in &prepared.expected_excluded { + let full = excluded.trim_end_matches('/'); + let relative = if prepared.prefix.is_empty() { + full + } else { + let Some(path) = full.strip_prefix(&format!("{}/", prepared.prefix)) else { + continue; + }; + path + }; + super::tree::safe_path(relative).map_err(RestoreError::Failed)?; + if self.check_source_path(relative).is_err() { + continue; + } + let parent = self.parent(relative).map_err(|_| RestoreError::Stale)?; + let leaf = Path::new(relative) + .file_name() + .ok_or_else(|| failed("Invalid excluded Git path"))?; + let metadata = parent + .symlink_metadata(leaf) + .map_err(|_| RestoreError::Stale)?; + if (metadata.is_dir() && !metadata.file_type().is_symlink()) != excluded.ends_with('/') + { + return Err(RestoreError::Stale); + } + } + Ok(()) + } +} +fn generation(dir: &Dir) -> Result { + let file = dir.try_clone().map_err(failed)?.into_std_file(); + super::ownership::generation(&file).map_err(RestoreError::Failed) +} +fn failed(error: impl std::fmt::Display) -> RestoreError { + RestoreError::Failed(fail(error)) +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index d5b8f639a..6e0b1984b 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,5 +1,7 @@ //! Location-bound native recovery journals and private non-Git BLAKE3 snapshots. mod git; +mod git_rules; +mod inventory; mod links; mod ownership; mod preparation; diff --git a/crates/cyber-snapshot/src/copy/tree.rs b/crates/cyber-snapshot/src/copy/tree.rs index 1862da812..6c27670b1 100644 --- a/crates/cyber-snapshot/src/copy/tree.rs +++ b/crates/cyber-snapshot/src/copy/tree.rs @@ -11,8 +11,10 @@ use std::path::{Component, Path}; pub(super) fn safe_path(path: &str) -> Result<()> { if path.is_empty() || path.len() > 4096 - || path.contains('\\') || path.contains('\0') + || path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") || Path::new(path) .components() .any(|c| !matches!(c, Component::Normal(_))) diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index bb94b1e9f..56862ff7e 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -23,6 +23,12 @@ pub(crate) struct PreparedTree { pub images: BTreeMap, } +pub(crate) struct IgnoreInputs { + pub info: PathBuf, + pub global: Option, + pub case_insensitive: bool, +} + pub(crate) struct PreparedRestore { pub directory: PathBuf, pub worktree: PathBuf, @@ -31,7 +37,11 @@ pub(crate) struct PreparedRestore { pub recorded: PreparedTree, pub expected: PreparedTree, pub exclusions: Vec, + /// The expected snapshot's exact worktree-relative exclusions, not globs. + /// Historical exclusions constrain mutation, but not the current-state check. + pub expected_excluded: Vec, pub allowed: Option>, + pub ignore: IgnoreInputs, } impl Shadow { @@ -42,6 +52,51 @@ impl Shadow { max_file: u64, max_total: u64, ) -> Result { + let default_global = std::env::var_os("XDG_CONFIG_HOME") + .filter(|value| !value.is_empty()) + .map(|path| PathBuf::from(path).join("git/ignore")) + .unwrap_or_else(|| PathBuf::from("~/.config/git/ignore")); + let configured = self + .run_bounded( + &[ + "config", + "--null", + "--path", + "--default", + default_global + .to_str() + .ok_or("Invalid global ignore path")?, + "--get", + "core.excludesfile", + ], + 8192, + ) + .await?; + let global = std::str::from_utf8( + configured + .strip_suffix(&[0]) + .ok_or("Invalid global ignore setting")?, + ) + .map_err(|_| "Non-UTF-8 global ignore path is unsupported")?; + let global = (!global.is_empty()).then(|| self.worktree.join(global)); + let ignore_case = self + .run_bounded( + &[ + "config", + "--bool", + "--default", + "false", + "--get", + "core.ignorecase", + ], + 16, + ) + .await?; + let ignore = IgnoreInputs { + info: self.git_dir.join("info/exclude"), + global, + case_insensitive: ignore_case == b"true\n", + }; let prefix = if self.pathspec == "." { "" } else { @@ -56,6 +111,10 @@ impl Shadow { for id in ids { excluded.extend(self.recorded_exclusions(id).await?.unwrap_or_default()); } + let expected_excluded = self + .recorded_exclusions(&request.expected) + .await? + .unwrap_or_default(); let exclusions = self .historical_exclusions(&ids.map(|id| id.to_owned())) .await?; @@ -71,10 +130,15 @@ impl Shadow { .transpose()?; let mut remaining = max_total; let mut trees = Vec::new(); - for id in ids { + for (index, id) in ids.into_iter().enumerate() { let mut entries = BTreeMap::new(); for (path, (mode, object)) in self.image_catalog(id).await?.iter() { - if history::excludes(&excluded, path) { + let omitted = if index == 2 { + history::excludes(&expected_excluded, path) + } else { + history::excludes(&excluded, path) + }; + if omitted { continue; } if !matches!(mode.as_str(), "100644" | "100755" | "120000") { @@ -115,7 +179,9 @@ impl Shadow { recorded: trees.next().ok_or("Missing recorded images")?, expected: trees.next().ok_or("Missing expected images")?, exclusions, + expected_excluded, allowed, + ignore, }) } } diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index edd8a446f..ad5eda18d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1883,3 +1883,71 @@ an initial Python-only probe was not equivalent and is not claimed as a native resolver defect. One test initially expected only the identity-check error text; it was corrected to accept the earlier protected-path refusal as well. The final full snapshot run and three-package all-target Clippy pass include these checks. + + +## Native Git namespace and rule-input validation (2026-11-07) + +`copy/inventory.rs` now scans the eligible Location namespace before source content +planning. It compares all eligible leaf names with the expected image set, so new +files, nested files and links cannot evade validation merely because they were +absent from every prepared tree. Unknown leaf kinds are not opened and source +directory links are not followed. Empty directories do not become Git file images. +Known expected exclusion paths are checked for disappearance and directory/leaf +type changes. Traversal is bounded by 65,536 nodes, depth 64 and 8 MiB of path/mark +metadata. Directory generations are checked during scanning and again after +content planning, before creating the intent. These marks are transient race +checks, not native source/recovery ownership. + +`copy/git_rules.rs` uses bounded read-only global/info/ancestor/nested Git ignore +inputs, including negation precedence and Git's case setting. The trusted adapter +resolves configuration locators before dispatch; source and configuration reads +remain in the synchronous filesystem worker, with no Git process inside it. Each +input is limited to 1 MiB, combined inputs/path metadata to 8 MiB and bindings to +65,536. Input hashes are revalidated at the end of planning. Source rule traversal +uses retained worktree descriptors and no-follow descendant opens; external Git +configuration files retain their existing read-only Git semantics. Their bytes are +used only for matching, never source mutation authority or model instructions. +Pattern errors use generic diagnostics rather than echoing private patterns. + +Testing with actual Git established that `ls-files --ignored --directory` can +compact an all-ignored directory and omit ignored children in a mixed directory. +Recorded exclusion paths alone therefore cannot classify current eligibility. The +new scan uses Git rules instead of assuming every unrecorded file is eligible or +blindly treating a compacted historical directory as permanently opaque. + +Prepared target/recorded images still apply the complete historical exclusion +union. Expected images now use only their own exclusions, allowing stale-state +checks for a currently eligible owner file that was excluded historically while +keeping it out of the mutation plan. Shared native path validation now accepts +literal Unix backslashes and explicitly rejects empty/dot/traversal components. +Git Location/file-name and actual non-Git capture/restore regressions exercise this. + +Nine additional tests cover new leaf/nested/link refusal, mixed ignored directories, +empty directory preservation, namespace/rule changes after inventory, historical +exclusion preservation versus current-state validation, subdirectory ancestor and +negation rules, global/info inputs and rule-only changes, known exclusion deletion/ +type changes, and literal backslashes. All source refusals retain current files and +create no intent. The existing 113 snapshot tests remain included. + +The code-review skill was applied to all source, test and spec changes, focusing on +no-follow input traversal, bounded parsing/inventory, readonly configuration roles, +separate mutation/state exclusions and preservation of the admission gate. No +critical/major issue remains in this disabled inventory scope. It does not establish +a full current-tag proof: version-1 tags lack capture-bound rule/raw-state data, +and ignored-inventory identity can differ from eligible-graph identity. Attributes/ +filters, gitlinks, file/directory transitions, capture-bound configuration, actual +Git lifetime and runtime acknowledgement integration remain required. The ordinary +Git availability/reservation gates remain disabled and parent tasks remain open. + +Verification on local macOS arm64: locked snapshot tests passed 122 cases with two +ignored fixtures; the pathspec fixture is invoked by its parent. Snapshot all-target +Clippy passed with warnings denied after handling the ignore builder's fallible +case-policy setter. The first scan-only build's unused import was corrected. Full +workspace tests, SDK execution, real CLI/TUI/App acceptance and Linux/Windows +execution were not run for this increment. Final format/diff, SDK-generation and +OpenSpec validation are recorded below. No API/schema shape changed here. + +Final formatting/diff checks and generated SDK consistency passed. Strict pinned +OpenSpec passed all 117 items; spec lint reports 0 errors/25 warnings. The real-Git +compaction probe and current tests are local evidence only; no CI or complete Git +recovery admission is claimed. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index d7e139935..3141183d8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -497,3 +497,32 @@ bytes under the original native observation before independent intent preparatio New Git-desired links must stay inside the worktree and outside protected/private roots; rollback may reinstate an exact owned preimage without following it. This does not authorize broad Git admission before full state/lifetime/acceptance gates. + + +### Native pre-effect Git namespace inventory + +The synchronous Git importer inventories the eligible Location namespace rather +than only names present in prepared trees. Current Git rule inputs cover the global +exclude file, shadow info/exclude, ancestor .gitignore files and nested .gitignore +files; configuration locators come from the trusted adapter, never a recovery +journal. These read-only configuration inputs reproduce existing Git reads and +grant no source mutation or recovery authority. Source rule files are traversed +without following directory/file links; Git-ignored .gitignore symlinks contribute +no rules. Inputs and traversal are bounded, and input hashes are checked again +before intent publication. Protected/private source namespaces remain excluded. + +Historical exclusions constrain target/recorded mutation separately from the +expected snapshot's own exclusions and images. Formerly excluded files that are +currently eligible therefore still participate in stale-state detection without +becoming mutation targets. Known expected exclusions are checked for disappearance +and file/directory changes. Current eligible leaf names are compared with expected +names; empty directories do not invent Git images. Directory-generation marks are +revalidated after content planning and before intent publication. None of these +read-only marks grants native mutation, acknowledgement or reclamation authority. + +Version-1 tags do not bind all rule inputs or raw working-tree representations to +the capture epoch. Git's directory-compacted exclusion list is not a complete +ignore oracle, and a changed ignored inventory can change a tag without changing +its eligible graph. Snapshot-bound policy/raw-state proof, complete exclusion +identity parity, attribute/filter and remaining Git compatibility, lifetime and +acceptance gates are still required before admitting ordinary Git workers. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 02d4f1cd5..435e1c401 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -152,6 +152,13 @@ - **AND** review SHALL identify file versus symbolic-link images, while legacy images default to files - **AND** link-capable journals SHALL use a distinct version/backend binding without promoting older file-only ownership or changing historical file receipt digests +#### Scenario: Git namespace changes before native planning completes +- **WHEN** a native Git restore validates its prepared expected state +- **THEN** it SHALL compare the eligible Location leaf inventory, including newly appearing paths, without following directory links or opening unknown leaf kinds +- **AND** namespace generations and read-only Git rule inputs SHALL be revalidated before intent publication +- **AND** historical exclusions SHALL constrain mutation independently of the current expected-state check +- **AND** validation marks SHALL NOT be used as source mutation, acknowledgement or reclamation ownership + #### Scenario: Retained native checkout routes change - **WHEN** a snapshot worker's retained checkout claim, activity namespace or worktree creation record changes after acquisition - **THEN** the worker SHALL refuse source callbacks and consuming acknowledgement after verifying exact native file/directory identities and creation contents diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 6f720d1f8..118b5b311 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -292,3 +292,14 @@ classification without progress/replay; these are backend fixtures, not complete Git runtime acceptance. Ordinary Git admission remains disabled for full expected state/Git compatibility, lifetime/acknowledgement, retention, clients and complete crash/platform gates. M2–M8 and parent tasks remain open. + + +Native inventory continuation: the Git importer now compares eligible Location +leaf names, revalidates directory namespace generations and read-only Git rule +inputs, and rejects new/removed/type-changed state before intent publication. +Current expected images use their own exclusions; historical exclusions still +constrain target/recorded mutations. Native relative paths now preserve Unix +backslashes while rejecting empty/dot/traversal components. Snapshot-bound ignore/ +raw-state proof, full Git compatibility, actual Git lifetime/acknowledgement, safe +reclamation and full client/platform acceptance remain open. Admission remains +disabled; no M2–M8 or parent task is accepted by these checks. From e6fc670d8e9a4e53ed6fc2663b8f2e9dfe06f79d Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 03:53:56 -0400 Subject: [PATCH 24/87] Bind Git checkpoint policy inputs to capture-time fingerprints --- crates/cyber-snapshot/src/copy/git.rs | 6 + crates/cyber-snapshot/src/copy/git/tests.rs | 163 +++++++++++++++- crates/cyber-snapshot/src/copy/git_rules.rs | 178 ++++++++++++++++-- crates/cyber-snapshot/src/copy/inventory.rs | 13 ++ crates/cyber-snapshot/src/copy/mod.rs | 2 +- crates/cyber-snapshot/src/git.rs | 55 +++++- crates/cyber-snapshot/src/git/catalog.rs | 6 + crates/cyber-snapshot/src/git/prepared.rs | 36 +++- crates/cyber-snapshot/src/git_history.rs | 105 ++++++++++- crates/cyber-snapshot/src/lib.rs | 16 +- .../implementation.md | 45 +++++ .../managed-recovery.md | 15 ++ .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 11 ++ 14 files changed, 623 insertions(+), 34 deletions(-) diff --git a/crates/cyber-snapshot/src/copy/git.rs b/crates/cyber-snapshot/src/copy/git.rs index 1b1571b6c..920b5daf3 100644 --- a/crates/cyber-snapshot/src/copy/git.rs +++ b/crates/cyber-snapshot/src/copy/git.rs @@ -115,6 +115,12 @@ impl Store { "Git handoff belongs to another native Location".into(), )); } + if prepared.expected_policy.is_none() { + return Err(RestoreError::Failed( + "Expected Git checkpoint lacks capture-bound policy proof; refresh the preview" + .into(), + )); + } self.ensure_recovered().map_err(RestoreError::Failed)?; // Known image validation precedes any source effect. Full state scanning // and actual runtime lifetime integration remain required before activation. diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index 6b034443c..cc78f4da4 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -117,7 +117,8 @@ async fn native_git_import_restores_files_modes_creations_deletions_and_retains_ .await; let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); // Native execution must not ask Git for object bytes or use its index again. - std::fs::rename(&shadow.git_dir, shadow.git_dir.with_extension("retained")).unwrap(); + let objects = shadow.git_dir.join("objects"); + std::fs::rename(&objects, objects.with_extension("retained")).unwrap(); let store = repo.store(&repo.root); assert_eq!( store.restore_git_worker(prepared, None).unwrap(), @@ -926,3 +927,163 @@ async fn non_git_snapshots_capture_and_restore_literal_backslash_names() { b"before" ); } + +#[tokio::test] +async fn capture_bound_policy_rejects_ignored_inventory_changes_before_planning() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitignore"), b"*.log\n").unwrap(); + std::fs::create_dir(repo.root.join("mixed")).unwrap(); + std::fs::write(repo.root.join("mixed/kept"), b"owner").unwrap(); + std::fs::write(repo.root.join("mixed/old.log"), b"ignored owner").unwrap(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + std::fs::write(repo.root.join("mixed/new.log"), b"new ignored owner").unwrap(); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + assert!(prepared.expected_policy.is_some()); + let store = repo.store(&repo.root); + assert!(matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + )); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn capture_bound_policy_rejects_excluded_rule_changes_that_do_not_change_the_tree() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".git/info/exclude"), b".gitignore\n").unwrap(); + std::fs::write(repo.root.join(".gitignore"), b"*.log\n").unwrap(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + std::fs::write( + repo.root.join(".gitignore"), + b"*.log\n# changed ignored rule input\n", + ) + .unwrap(); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + assert!(!prepared.expected.images.contains_key(".gitignore")); + let store = repo.store(&repo.root); + assert!(matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + )); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn legacy_expected_policy_absence_never_authorizes_native_import() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + let mut prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + prepared.expected_policy = None; + let store = repo.store(&repo.root); + assert!( + matches!(store.restore_git_worker(prepared, None), Err(RestoreError::Failed(error)) if error.contains("capture-bound policy")) + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn capture_bound_policy_rejects_external_setting_changes_before_preparation() { + for change in ["content", "locator", "case"] { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + repo.track(&repo.root).await; + let global = repo.root.parent().unwrap().join("ignore-a"); + let other = repo.root.parent().unwrap().join("ignore-b"); + std::fs::write(&global, b"*.log\n").unwrap(); + std::fs::write(&other, b"*.log\n").unwrap(); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + let config = |key: &str, value: &str| { + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args(["config", key, value]) + .status() + .unwrap() + .success() + ); + }; + config("core.excludesfile", global.to_str().unwrap()); + config("core.ignorecase", "false"); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + match change { + "content" => std::fs::write(&global, b"*.log\n# changed input\n").unwrap(), + "locator" => config("core.excludesfile", other.to_str().unwrap()), + _ => config("core.ignorecase", "true"), + } + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert!( + matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + ), + "{change}" + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} + +#[tokio::test] +async fn capture_bound_policy_rejects_application_ignore_and_size_setting_changes() { + use std::sync::atomic::{AtomicBool, Ordering}; + for choice in ["ignore", "size"] { + let mut repo = Repo::new(); + let changed = Arc::new(AtomicBool::new(false)); + let config_changed = changed.clone(); + repo.snapshots = crate::GitSnapshots::new( + repo.data.clone(), + Arc::new(move |_| { + if !config_changed.load(Ordering::SeqCst) { + return serde_json::json!({}); + } + if choice == "ignore" { + serde_json::json!({"snapshots":{"ignore":["never-matched-name"]}}) + } else { + serde_json::json!({"snapshots":{"max_file_bytes":1048576}}) + } + }), + ); + std::fs::write(repo.root.join("file"), b"before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), b"after").unwrap(); + let recorded = repo.track(&repo.root).await; + changed.store(true, Ordering::SeqCst); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert!( + matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + ), + "{choice}" + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs index a5ce09690..5060e9aaf 100644 --- a/crates/cyber-snapshot/src/copy/git_rules.rs +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -1,6 +1,6 @@ //! Read-only Git rule inputs. These bindings grant no mutation or recovery authority. use super::storage::fail; -use crate::git::prepared::PreparedRestore; +use crate::git::prepared::{IgnoreInputs, PreparedRestore}; use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, OpenOptions, OpenOptionsExt}; use cyber_server::runtime::RestoreError; @@ -10,6 +10,7 @@ use std::path::{Path, PathBuf}; const INPUT_LIMIT: u64 = 1024 * 1024; const TOTAL_LIMIT: usize = 8 * 1024 * 1024; +#[derive(Clone, serde::Serialize)] struct Binding { path: PathBuf, source: bool, @@ -22,25 +23,24 @@ pub(super) struct Rules { bindings: Vec, stack: Vec, bytes: usize, + inputs: IgnoreInputs, } impl Rules { pub(super) fn new(prepared: &PreparedRestore) -> Result { + Self::open(&prepared.worktree, &prepared.prefix, &prepared.ignore) + } + fn open(worktree: &Path, prefix: &str, inputs: &IgnoreInputs) -> Result { let mut rules = Self { - root: Dir::open_ambient_dir(&prepared.worktree, cap_std::ambient_authority()) - .map_err(failed)?, - path: prepared.worktree.clone(), - case_insensitive: prepared.ignore.case_insensitive, + root: Dir::open_ambient_dir(worktree, cap_std::ambient_authority()).map_err(failed)?, + path: worktree.to_path_buf(), + case_insensitive: inputs.case_insensitive, bindings: vec![], stack: vec![], bytes: 0, + inputs: inputs.clone(), }; let mut builder = rules.builder()?; - for path in prepared - .ignore - .global - .iter() - .chain(std::iter::once(&prepared.ignore.info)) - { + for path in inputs.global.iter().chain(std::iter::once(&inputs.info)) { if let Some(bytes) = rules.bind(path, false)? { rules.lines(&mut builder, None, &bytes)?; } @@ -50,9 +50,9 @@ impl Rules { .build() .map_err(|_| failed("Unsupported Git ignore rules"))?, ); - if !prepared.prefix.is_empty() { + if !prefix.is_empty() { rules.enter("")?; - let mut parent = Path::new(&prepared.prefix).parent(); + let mut parent = Path::new(prefix).parent(); let mut ancestors = Vec::new(); while let Some(path) = parent.filter(|path| !path.as_os_str().is_empty()) { ancestors.push(path.to_path_buf()); @@ -131,6 +131,29 @@ impl Rules { }); Ok(bytes) } + pub(super) fn fingerprint( + &self, + ignored: &std::collections::BTreeSet, + ) -> Result { + let mut bindings: Vec<_> = self + .bindings + .iter() + .filter(|binding| !binding.source || binding.hash.is_some()) + .cloned() + .collect(); + bindings.sort_by(|left, right| { + (left.source, &left.path, &left.hash).cmp(&(right.source, &right.path, &right.hash)) + }); + let bytes = serde_json::to_vec(&( + "git-policy-capture-v1", + &self.path, + &self.inputs, + bindings, + ignored, + )) + .map_err(failed)?; + Ok(blake3::hash(&bytes).to_hex().to_string()) + } pub(super) fn verify(&self) -> Result<(), RestoreError> { for binding in &self.bindings { let bytes = read(&self.root, &binding.path, binding.source)?; @@ -203,3 +226,132 @@ fn read(root: &Dir, path: &Path, source: bool) -> Result>, Restor fn failed(error: impl std::fmt::Display) -> RestoreError { RestoreError::Failed(fail(error)) } + +/// Collect only read-only policy/state inputs. This is not native effect ownership. +pub(crate) fn capture_policy( + worktree: &Path, + prefix: &str, + inputs: &IgnoreInputs, +) -> Result { + let mut capture = PolicyCapture { + rules: Rules::open(worktree, prefix, inputs)?, + ignored: std::collections::BTreeSet::new(), + marks: vec![], + nodes: 0, + bytes: 0, + }; + let mut directory = capture.rules.root.try_clone().map_err(failed)?; + for part in Path::new(prefix).components() { + directory = directory + .open_dir_nofollow(part.as_os_str()) + .map_err(failed)?; + } + capture.walk(&directory, "", prefix, 0)?; + for (path, before) in &capture.marks { + let mut current = directory.try_clone().map_err(failed)?; + for part in Path::new(path).components() { + current = current + .open_dir_nofollow(part.as_os_str()) + .map_err(|_| RestoreError::Stale)?; + } + if dir_generation(¤t)? != *before { + return Err(RestoreError::Stale); + } + } + capture.rules.verify()?; + capture.rules.fingerprint(&capture.ignored) +} +struct PolicyCapture { + rules: Rules, + ignored: std::collections::BTreeSet, + marks: Vec<(String, String)>, + nodes: usize, + bytes: usize, +} +impl PolicyCapture { + fn walk( + &mut self, + dir: &Dir, + path: &str, + prefix: &str, + depth: usize, + ) -> Result<(), RestoreError> { + if depth > 64 { + return Err(failed("Git policy directory depth exceeds limit")); + } + let scope = if prefix.is_empty() { + path.to_owned() + } else if path.is_empty() { + prefix.into() + } else { + format!("{prefix}/{path}") + }; + self.rules.enter(&scope)?; + let before = dir_generation(dir)?; + self.charge(path.len() + before.len())?; + for entry in dir.entries().map_err(failed)? { + let leaf = entry.map_err(failed)?.file_name(); + let name = leaf + .to_str() + .ok_or_else(|| failed("Non-UTF-8 Git policy path is unsupported"))?; + let relative = if path.is_empty() { + name.to_owned() + } else { + format!("{path}/{name}") + }; + super::tree::safe_path(&relative).map_err(RestoreError::Failed)?; + self.charge(relative.len())?; + let full = if prefix.is_empty() { + relative.clone() + } else { + format!("{prefix}/{relative}") + }; + if super::tree::protected(Path::new(&relative)) + || self + .rules + .inputs + .private_roots + .iter() + .any(|root| self.rules.path.join(&full).starts_with(root)) + { + continue; + } + let metadata = dir.symlink_metadata(&leaf).map_err(failed)?; + let directory = metadata.is_dir() && !metadata.file_type().is_symlink(); + if self.rules.ignored(&full, directory) { + self.ignored + .insert(if directory { format!("{full}/") } else { full }); + } else if directory { + // Embedded repositories need their own opaque Git-state proof. + let child = dir + .open_dir_nofollow(&leaf) + .map_err(|_| RestoreError::Stale)?; + if child.symlink_metadata(".git").is_ok() { + return Err(failed("Embedded Git policy proof is unavailable")); + } + self.walk(&child, &relative, prefix, depth + 1)?; + } + } + if dir_generation(dir)? != before { + return Err(RestoreError::Stale); + } + self.rules.leave(); + self.marks.push((path.into(), before)); + Ok(()) + } + fn charge(&mut self, bytes: usize) -> Result<(), RestoreError> { + self.nodes += 1; + self.bytes = self + .bytes + .checked_add(bytes) + .ok_or_else(|| failed("Git policy capture size overflow"))?; + if self.nodes > 65536 || self.bytes > TOTAL_LIMIT { + return Err(failed("Git policy capture exceeds limit")); + } + Ok(()) + } +} +fn dir_generation(dir: &Dir) -> Result { + super::ownership::generation(&dir.try_clone().map_err(failed)?.into_std_file()) + .map_err(RestoreError::Failed) +} diff --git a/crates/cyber-snapshot/src/copy/inventory.rs b/crates/cyber-snapshot/src/copy/inventory.rs index 69965f709..90f2659b5 100644 --- a/crates/cyber-snapshot/src/copy/inventory.rs +++ b/crates/cyber-snapshot/src/copy/inventory.rs @@ -21,6 +21,7 @@ pub(super) struct Inventory { bytes: usize, found: BTreeSet, rules: super::git_rules::Rules, + ignored: BTreeSet, } impl Inventory { @@ -103,6 +104,12 @@ impl Inventory { &full, metadata.is_dir() && !metadata.file_type().is_symlink(), ) { + self.ignored + .insert(if metadata.is_dir() && !metadata.file_type().is_symlink() { + format!("{full}/") + } else { + full + }); continue; } if metadata.is_dir() && !metadata.file_type().is_symlink() { @@ -144,12 +151,18 @@ impl Store { bytes: 0, found: BTreeSet::new(), rules: super::git_rules::Rules::new(prepared)?, + ignored: BTreeSet::new(), }; inventory.walk(self, prepared, &self.work, "", 0)?; if !inventory.found.iter().eq(prepared.expected.images.keys()) { return Err(RestoreError::Stale); } inventory.verify(self)?; + if let Some(expected) = &prepared.expected_policy + && &inventory.rules.fingerprint(&inventory.ignored)? != expected + { + return Err(RestoreError::Stale); + } Ok(inventory) } fn check_expected_exclusions(&self, prepared: &PreparedRestore) -> Result<(), RestoreError> { diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 6e0b1984b..898c66baf 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,6 +1,6 @@ //! Location-bound native recovery journals and private non-Git BLAKE3 snapshots. mod git; -mod git_rules; +pub(crate) mod git_rules; mod inventory; mod links; mod ownership; diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 049a1f6f0..d521341c6 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -23,6 +23,9 @@ pub(crate) struct Shadow { /// The Location relative to the worktree, as a pathspec (`.` for the root). pathspec: String, catalogs: Arc>, + pub private_roots: Vec, + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub capture_settings: prepared::CaptureSettings, } /// Environment variables that would redirect git away from the shadow repository. @@ -96,6 +99,12 @@ impl Shadow { common_dir: git_path(&common_dir), pathspec: if rel.is_empty() { ".".into() } else { rel }, catalogs: Arc::new(Mutex::new(catalog::CatalogCache::new())), + private_roots: vec![root.parent().unwrap_or(root).to_path_buf()], + #[cfg(any(target_os = "macos", target_os = "linux"))] + capture_settings: prepared::CaptureSettings { + ignore: vec![], + max_untracked: 2 * 1024 * 1024, + }, }) } @@ -161,6 +170,8 @@ impl Shadow { self.ensure_init().await?; let skipped = self.large_untracked(max_bytes).await; self.write_excludes(ignore, &skipped)?; + #[cfg(any(target_os = "macos", target_os = "linux"))] + let policy_before = self.capture_policy(ignore, max_bytes).await; let excluded = self.omitted().await?; let cached_paths: Vec<_> = excluded .iter() @@ -179,10 +190,52 @@ impl Shadow { .trim() .to_string(); let skipped = excluded.clone(); - let tree = self.record_snapshot(&tree, excluded).await?; + #[cfg(any(target_os = "macos", target_os = "linux"))] + let (policy, skipped) = match policy_before { + Ok(before) => { + let after = self.capture_policy(ignore, max_bytes).await?; + if before != after { + return Err("Git policy changed during checkpoint capture".into()); + } + (Some(after), skipped) + } + Err(_) => { + let mut skipped = skipped; + skipped.push("native Git policy proof unavailable for this capture".into()); + (None, skipped) + } + }; + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + let policy = None; + let tree = self.record_snapshot(&tree, excluded, policy).await?; Ok(Snapshot { tree, skipped }) } + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn capture_policy( + &self, + ignore: &[String], + max_untracked: u64, + ) -> Result { + let mut inputs = self.ignore_inputs().await?; + inputs.settings = prepared::CaptureSettings { + ignore: ignore.to_vec(), + max_untracked, + }; + let worktree = self.worktree.clone(); + let prefix = if self.pathspec == "." { + String::new() + } else { + self.pathspec.clone() + }; + tokio::task::spawn_blocking(move || { + crate::copy::git_rules::capture_policy(&worktree, &prefix, &inputs) + }) + .await + .map_err(|error| error.to_string())? + .map_err(|error| format!("{error:?}")) + } + /// Untracked files (per the user's repository) over the size limit. async fn large_untracked(&self, max_bytes: u64) -> Vec { let mut cmd = tokio::process::Command::new("git"); diff --git a/crates/cyber-snapshot/src/git/catalog.rs b/crates/cyber-snapshot/src/git/catalog.rs index 499c74169..1e61a79a8 100644 --- a/crates/cyber-snapshot/src/git/catalog.rs +++ b/crates/cyber-snapshot/src/git/catalog.rs @@ -169,6 +169,12 @@ mod tests { common_dir: temp.path().join("common"), pathspec: ".".into(), catalogs: Default::default(), + private_roots: vec![], + #[cfg(any(target_os = "macos", target_os = "linux"))] + capture_settings: super::super::prepared::CaptureSettings { + ignore: vec![], + max_untracked: 2 * 1024 * 1024, + }, }; assert!( shadow diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index 56862ff7e..4a849c5f9 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -23,10 +23,19 @@ pub(crate) struct PreparedTree { pub images: BTreeMap, } +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub(crate) struct CaptureSettings { + pub ignore: Vec, + pub max_untracked: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] pub(crate) struct IgnoreInputs { pub info: PathBuf, pub global: Option, pub case_insensitive: bool, + pub private_roots: Vec, + pub settings: CaptureSettings, } pub(crate) struct PreparedRestore { @@ -42,16 +51,11 @@ pub(crate) struct PreparedRestore { pub expected_excluded: Vec, pub allowed: Option>, pub ignore: IgnoreInputs, + pub expected_policy: Option, } impl Shadow { - /// No source reads, Git index writes or merge subprocesses occur here. - pub(crate) async fn prepare_restore( - &self, - request: &SnapshotRestoreRequest, - max_file: u64, - max_total: u64, - ) -> Result { + pub(crate) async fn ignore_inputs(&self) -> Result { let default_global = std::env::var_os("XDG_CONFIG_HOME") .filter(|value| !value.is_empty()) .map(|path| PathBuf::from(path).join("git/ignore")) @@ -92,11 +96,24 @@ impl Shadow { 16, ) .await?; - let ignore = IgnoreInputs { + Ok(IgnoreInputs { info: self.git_dir.join("info/exclude"), global, case_insensitive: ignore_case == b"true\n", - }; + private_roots: self.private_roots.clone(), + settings: self.capture_settings.clone(), + }) + } + + /// No source reads, Git index writes or merge subprocesses occur here. + pub(crate) async fn prepare_restore( + &self, + request: &SnapshotRestoreRequest, + max_file: u64, + max_total: u64, + ) -> Result { + let ignore = self.ignore_inputs().await?; + let expected_policy = self.snapshot_policy(&request.expected).await?; let prefix = if self.pathspec == "." { "" } else { @@ -182,6 +199,7 @@ impl Shadow { expected_excluded, allowed, ignore, + expected_policy, }) } } diff --git a/crates/cyber-snapshot/src/git_history.rs b/crates/cyber-snapshot/src/git_history.rs index 537e1a79e..ee67ad96c 100644 --- a/crates/cyber-snapshot/src/git_history.rs +++ b/crates/cyber-snapshot/src/git_history.rs @@ -15,6 +15,8 @@ struct Record { location: PathBuf, tree: String, excluded: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + policy: Option, } fn oid(id: &str) -> bool { matches!(id.len(), 40 | 64) @@ -96,21 +98,31 @@ impl Shadow { &self, tree: &str, excluded: Vec, + policy: Option, ) -> Result { if !oid(tree) { return Err("Invalid snapshot tree identity".into()); } + if policy.as_ref().is_some_and(|hash| { + hash.len() != 64 + || !hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) { + return Err("Invalid Git policy fingerprint".into()); + } let record = Record { - version: 1, + version: if policy.is_some() { 2 } else { 1 }, location: self.worktree.join(&self.pathspec), tree: tree.into(), excluded, + policy, }; let bytes = serde_json::to_vec(&record).map_err(|e| e.to_string())?; if bytes.len() > LIMIT { return Err("Snapshot exclusions exceed record limit".into()); } - let mut tag = format!("object {tree}\ntype tree\ntag cyber-snapshot-v1\ntagger Cyber Snapshot 0 +0000\n\n").into_bytes(); + let mut tag = format!("object {tree}\ntype tree\ntag cyber-snapshot-v{}\ntagger Cyber Snapshot 0 +0000\n\n", record.version).into_bytes(); tag.extend(bytes); let id = String::from_utf8(self.input(&["mktag"], &tag).await?) .map_err(|e| e.to_string())? @@ -136,16 +148,27 @@ impl Shadow { /// Return the root identity from the same verified bytes as the exclusions. /// Resolving the tag again would allow a later substitution to change its root. + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub(super) async fn snapshot_policy(&self, id: &str) -> Result, String> { + Ok(self.read_record(id).await?.and_then(|record| record.policy)) + } pub(super) async fn snapshot_metadata( &self, id: &str, ) -> Result<(String, Option>), String> { + Ok(match self.read_record(id).await? { + Some(record) => (record.tree, Some(record.excluded)), + None => (id.to_owned(), None), + }) + } + + async fn read_record(&self, id: &str) -> Result, String> { if !oid(id) { return Err("Invalid snapshot identity".into()); } let kind = self.run(&["cat-file", "-t", id]).await?; if kind == b"tree\n" { - return Ok((id.to_owned(), None)); + return Ok(None); } if kind != b"tag\n" { return Err("Snapshot identity is not a tree or exclusion record".into()); @@ -168,21 +191,28 @@ impl Shadow { .split_once("\n\n") .ok_or("Invalid snapshot exclusion record")?; let record: Record = serde_json::from_str(body).map_err(|e| e.to_string())?; - if record.version != 1 - || record.location != self.worktree.join(&self.pathspec) + if !matches!( + (record.version, record.policy.as_deref()), + (1, None) | (2, Some(_)) + ) || record.policy.as_ref().is_some_and(|policy| { + policy.len() != 64 + || !policy + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) || record.location != self.worktree.join(&self.pathspec) || !oid(&record.tree) || record.excluded.len() > PATH_LIMIT || record.excluded.iter().any(|p| !safe_path(p)) || !header.starts_with(&format!( - "object {}\ntype tree\ntag cyber-snapshot-v1\n", - record.tree + "object {}\ntype tree\ntag cyber-snapshot-v{}\n", + record.tree, record.version )) { return Err( "Snapshot exclusion record belongs to another Location or is corrupt".into(), ); } - Ok((record.tree, Some(record.excluded))) + Ok(Some(record)) } pub(super) async fn excluded_between( &self, @@ -214,3 +244,62 @@ impl Shadow { Ok(out.into_iter().collect()) } } + +#[cfg(all(test, any(target_os = "macos", target_os = "linux")))] +mod policy_tests { + use super::*; + #[tokio::test] + async fn versioned_tags_keep_legacy_metadata_readable_without_promoting_policy_proof() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("work"); + std::fs::create_dir(&root).unwrap(); + assert!( + std::process::Command::new("git") + .args(["init", "--quiet"]) + .arg(&root) + .status() + .unwrap() + .success() + ); + std::fs::write(root.join("file"), b"source").unwrap(); + let shadow = Shadow::locate(&base.join("data/snapshot"), &root).unwrap(); + let checkpoint = shadow.track(&[], 2 * 1024 * 1024).await.unwrap(); + let record = shadow.read_record(&checkpoint.tree).await.unwrap().unwrap(); + assert_eq!(record.version, 2); + assert!(record.policy.is_some()); + let legacy = shadow + .record_snapshot(&record.tree, record.excluded.clone(), None) + .await + .unwrap(); + assert_eq!( + shadow.read_record(&legacy).await.unwrap().unwrap().version, + 1 + ); + assert!(shadow.snapshot_policy(&legacy).await.unwrap().is_none()); + assert!( + shadow + .snapshot_policy(&record.tree) + .await + .unwrap() + .is_none() + ); + assert_eq!( + shadow.blob(&legacy, "file").await.unwrap().unwrap().1, + b"source" + ); + assert!( + shadow + .record_snapshot(&record.tree, vec![], Some("invalid".into())) + .await + .is_err() + ); + let mut invalid = shadow.read_record(&legacy).await.unwrap().unwrap(); + invalid.policy = Some("a".repeat(64)); + let body = serde_json::to_vec(&invalid).unwrap(); + let mut tag = format!("object {}\ntype tree\ntag cyber-snapshot-v1\ntagger Cyber Snapshot 0 +0000\n\n", record.tree).into_bytes(); + tag.extend(body); + let id = String::from_utf8(shadow.input(&["mktag"], &tag).await.unwrap()).unwrap(); + assert!(shadow.snapshot_metadata(id.trim()).await.is_err()); + } +} diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index a6ac7f5f9..9aab51824 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -68,7 +68,21 @@ impl GitSnapshots { if (self.config)(location).get("snapshots") == Some(&Value::Bool(false)) { return None; } - Shadow::locate(&self.root(), location) + let mut shadow = Shadow::locate(&self.root(), location)?; + shadow + .private_roots + .extend(self.private_roots.iter().cloned()); + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + let (ignore, max_untracked) = self.options(location); + shadow.capture_settings = git::prepared::CaptureSettings { + ignore, + max_untracked, + }; + } + shadow.private_roots.sort(); + shadow.private_roots.dedup(); + Some(shadow) } fn non_git(&self, directory: &str) -> bool { diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index ad5eda18d..4002427e1 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1951,3 +1951,48 @@ Final formatting/diff checks and generated SDK consistency passed. Strict pinned OpenSpec passed all 117 items; spec lint reports 0 errors/25 warnings. The real-Git compaction probe and current tests are local evidence only; no CI or complete Git recovery admission is claimed. + + +## Capture-bound Git policy fingerprints (2026-11-07) + +Version-2 immutable Git tags now bind a BLAKE3 policy fingerprint collected before +and after Git tree capture. The fingerprint includes application ignore/size +settings, private roots, effective global ignore locator and case policy, bounded +rule input hashes, and canonical ignored names. Missing source rule files are +excluded from the persistent digest so empty-directory creation does not change +Git file identity, but their absence is still checked during each transient scan. +Unsupported initial collection retains a legacy capture with a generic unavailable +proof diagnostic; a changed or failed second collection refuses publication. +Legacy version-1 tags and bare trees remain readable without policy proof. Native +import refuses an expected checkpoint lacking proof before recovery/journal access. + +Five new native importer regressions cover ignored-name changes in mixed dirs, +ignored rule-file edits, absent legacy proof, external content/locator/case changes, +and application ignore/size setting changes. One history regression exercises v2, +v1 and bare-tree reading, invalid writer fingerprints and forged v1 policy fields. +The native object independence fixture now removes only the Git object directory: +removing all shadow metadata correctly invalidates its bound info/exclude input. + +The code-review skill was applied to every changed source/test file and artifacts. +Review covered cfg boundaries, tag framing and typed object verification, bounded +no-follow traversal, deterministic fingerprints, configuration versus effect +ownership, unsupported fallback, and absence of an admission-gate bypass. No +critical/major finding remains for this disabled policy-validation increment. +The effective-setting resolver's source configuration files are not bound across +resolution/worker dispatch. Raw file bytes, attributes/filters, gitlinks and complete +exclusion parity are not proven by this fingerprint. Existing Git capture may run +configured clean filters. This work does not establish process isolation, Git +worker lifetime, post-effect acknowledgement or reclamation authority. + +Local macOS arm64 verification: locked snapshot tests passed 128 cases, with two +ignored fixtures (the pathspec fixture is invoked by its parent). Snapshot +all-target Clippy passed with warnings denied. An earlier run exposed the obsolete +whole-shadow-removal expectation, corrected above; an earlier Clippy run found a +collapsible conditional, corrected before the passing run. No full workspace, +Linux/Windows, real CLI/TUI/App or source-bound CI acceptance was run for this +increment. Ordinary Git availability/reservation gates stay disabled and all +parent recovery tasks remain open. + +Final format/diff checks and SDK generation consistency passed. Pinned strict +OpenSpec validation passed all 117 items; spec lint reported 0 errors/25 warnings. +These results do not provide source-bound CI or complete recovery acceptance. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 3141183d8..39760bc42 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -526,3 +526,18 @@ ignore oracle, and a changed ignored inventory can change a tag without changing its eligible graph. Snapshot-bound policy/raw-state proof, complete exclusion identity parity, attribute/filter and remaining Git compatibility, lifetime and acceptance gates are still required before admitting ordinary Git workers. + + +### Capture-bound Git policy fingerprints + +New supported native captures bind a versioned fingerprint of the effective Git +ignore settings, read-only rule input hashes and canonical ignored namespace names +into an immutable version-2 checkpoint tag. Policy is collected before and after +Git capture; a changed successful collection refuses publication. Unsupported +collections retain a readable legacy checkpoint with an explicit unavailable +diagnostic, never fabricated policy proof. Version-1 tags and bare trees remain +readable without acquiring that evidence. Native inventory compares the current +fingerprint with the expected tag before journaling. This is a policy/state-input +precondition, not worker, mutation, acknowledgement or reclamation authority. Raw +working-tree representation and the remaining admission/compatibility gates remain +required; a policy fingerprint alone does not activate ordinary Git recovery. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 435e1c401..b629079de 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -241,6 +241,12 @@ - **THEN** later requests with its ID look up that original result or incomplete outcome and never dispatch source changes again - **AND** changed request content refuses, preimages and directory ownership evidence remain retained, and filesystem completion alone does not settle original/native fences +#### Scenario: Native Git policy is bound to the expected capture +- **WHEN** native Git restore validates an expected checkpoint +- **THEN** its immutable capture record SHALL bind effective ignore settings, bounded read-only rule input hashes and canonical ignored namespace names, and current inputs SHALL match before journal publication +- **AND** successful policy collections before and after capture SHALL agree; unsupported capture SHALL report unavailable proof without inventing it, and legacy tags/bare trees SHALL remain readable without acquiring native admission authority +- **AND** this policy fingerprint SHALL NOT substitute for raw working-tree state, exact worker ownership, native acknowledgement or reference-aware reclamation + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 118b5b311..5b9c864dc 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -303,3 +303,14 @@ backslashes while rejecting empty/dot/traversal components. Snapshot-bound ignor raw-state proof, full Git compatibility, actual Git lifetime/acknowledgement, safe reclamation and full client/platform acceptance remain open. Admission remains disabled; no M2–M8 or parent task is accepted by these checks. + + +Capture-policy continuation: version-2 Git checkpoint tags now retain bounded +capture-time policy fingerprints. Native inventory compares effective ignore +settings, rule hashes and canonical ignored names before intent publication; +legacy records remain readable without native proof. Captures bracket Git tree +creation with policy collections and refuse a changed successful collection. +Application ignore/size settings and private roots are included. Raw working-tree +representation, configuration-resolution races, Git compatibility, actual worker +lifetime/acknowledgement, reclamation and full clients/platform acceptance remain +open. Git admission stays disabled; M2–M8 and parent tasks remain incomplete. From 48d8736e2f8ca0b0cd0dbd7f17ce941d7723be67 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 04:03:22 -0400 Subject: [PATCH 25/87] Retain exact working-tree images in Git checkpoint graphs --- crates/cyber-snapshot/src/copy/git/tests.rs | 206 ++++++++++++++++++ crates/cyber-snapshot/src/copy/git_capture.rs | 62 ++++++ crates/cyber-snapshot/src/copy/git_rules.rs | 28 ++- crates/cyber-snapshot/src/copy/mod.rs | 1 + crates/cyber-snapshot/src/git.rs | 46 ++-- crates/cyber-snapshot/src/git/catalog.rs | 2 +- crates/cyber-snapshot/src/git/prepared.rs | 2 +- crates/cyber-snapshot/src/git/raw.rs | 67 ++++++ crates/cyber-snapshot/src/git_history.rs | 122 ++++++++++- crates/cyber-snapshot/tests/snapshots.rs | 43 ++-- .../implementation.md | 69 ++++++ .../managed-recovery.md | 14 ++ .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 11 + 14 files changed, 622 insertions(+), 57 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/git_capture.rs create mode 100644 crates/cyber-snapshot/src/git/raw.rs diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index cc78f4da4..fdf946229 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -1087,3 +1087,209 @@ async fn capture_bound_policy_rejects_application_ignore_and_size_setting_change assert!(store.recovery().unwrap().is_empty()); } } + +#[tokio::test] +async fn native_git_uses_exact_captured_crlf_bytes_and_keeps_both_graphs_reachable() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitattributes"), b"*.txt text eol=crlf\n").unwrap(); + std::fs::write(repo.root.join("file.txt"), b"before\r\n").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file.txt"), b"after\r\n").unwrap(); + let recorded = repo.track(&repo.root).await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + // Canonical public Git consumers still see the normalized object. + assert_eq!( + shadow.blob(&target, "file.txt").await.unwrap().unwrap().1, + b"before\n" + ); + // Aggressive ordinary Git GC must retain the raw graph through the tag ref. + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args(["gc", "--prune=now"]) + .status() + .unwrap() + .success() + ); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + assert_eq!(prepared.target.images["file.txt"].bytes, b"before\r\n"); + assert_eq!(prepared.expected.images["file.txt"].bytes, b"after\r\n"); + let store = repo.store(&repo.root); + assert_eq!( + store.restore_git_worker(prepared, None).unwrap(), + ["file.txt"] + ); + assert_eq!( + std::fs::read(repo.root.join("file.txt")).unwrap(), + b"before\r\n" + ); +} + +#[tokio::test] +async fn native_git_rejects_later_normalization_equivalent_owner_bytes() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitattributes"), b"*.txt text eol=lf\n").unwrap(); + std::fs::write(repo.root.join("file.txt"), b"before\n").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file.txt"), b"after\n").unwrap(); + let recorded = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file.txt"), b"after\r\n").unwrap(); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert!(matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + )); + assert_eq!( + std::fs::read(repo.root.join("file.txt")).unwrap(), + b"after\r\n" + ); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn native_git_raw_capture_supports_sha256_shadow_graphs_and_encoded_bytes() { + let repo = Repo::new(); + // Test the graph codec at both object widths. This does not claim public + // discovery of SHA-256 user repositories (libgit2 discovery still refuses it). + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + std::fs::create_dir_all(&shadow.git_dir).unwrap(); + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args(["init", "--bare", "--quiet", "--object-format=sha256"]) + .status() + .unwrap() + .success() + ); + std::fs::write( + repo.root.join(".gitattributes"), + b"*.txt working-tree-encoding=UTF-16LE\n", + ) + .unwrap(); + let before: Vec = "before\n" + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + let after: Vec = "after\n" + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + std::fs::write(repo.root.join("file.txt"), &before).unwrap(); + let target = repo.track(&repo.root).await; + assert_eq!(target.len(), 64); + std::fs::write(repo.root.join("file.txt"), &after).unwrap(); + let recorded = repo.track(&repo.root).await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + assert_eq!( + shadow.blob(&target, "file.txt").await.unwrap().unwrap().1, + b"before\n" + ); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + assert_eq!(prepared.expected.images["file.txt"].bytes, after); + let store = repo.store(&repo.root); + assert_eq!( + store.restore_git_worker(prepared, None).unwrap(), + ["file.txt"] + ); + assert_eq!(std::fs::read(repo.root.join("file.txt")).unwrap(), before); +} + +#[tokio::test] +async fn corrupt_raw_git_blob_cannot_be_promoted_to_native_restore_evidence() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitattributes"), b"*.txt text eol=crlf\n").unwrap(); + std::fs::write(repo.root.join("file.txt"), b"before\r\n").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file.txt"), b"after\r\n").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + let object = |id: &str| shadow.git_dir.join("objects").join(&id[..2]).join(&id[2..]); + let original = object(&prepared.target.images["file.txt"].object); + let replacement = std::fs::read(object(&prepared.recorded.images["file.txt"].object)).unwrap(); + std::fs::set_permissions(&original, std::fs::Permissions::from_mode(0o600)).unwrap(); + std::fs::write(original, replacement).unwrap(); + assert!( + shadow + .prepare_restore( + &SnapshotRestoreRequest { + directory: repo.root.to_str().unwrap().into(), + target, + recorded: recorded.clone(), + expected: recorded, + paths: None, + }, + 65536, + 8 * 1024 * 1024 + ) + .await + .is_err() + ); + assert_eq!( + std::fs::read(repo.root.join("file.txt")).unwrap(), + b"after\r\n" + ); + assert!(repo.store(&repo.root).recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn git_capture_does_not_publish_a_checkpoint_if_a_fixture_filter_changes_raw_state() { + let repo = Repo::new(); + repo.track(&repo.root).await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + let refs = || { + let output = std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args([ + "for-each-ref", + "--format=%(objectname)", + "refs/cyber/snapshots", + ]) + .output() + .unwrap(); + assert!(output.status.success()); + output.stdout + }; + let before = refs(); + // This explicitly configured fixture filter changes only its own temp repo. + // Canonical capture already executes Git filters; raw proof must not hide it. + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args([ + "config", + "filter.fixture.clean", + "printf changed > capture-marker; cat" + ]) + .status() + .unwrap() + .success() + ); + std::fs::write(repo.root.join(".gitattributes"), b"file filter=fixture\n").unwrap(); + std::fs::write(repo.root.join("file"), b"owner").unwrap(); + let result = repo.snapshots.track(repo.root.to_str().unwrap()).await; + assert!(matches!(result, Err(error) if error.contains("raw state changed"))); + assert_eq!( + refs(), + before, + "failed capture must not publish a checkpoint ref" + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"owner"); + assert_eq!( + std::fs::read(repo.root.join("capture-marker")).unwrap(), + b"changed" + ); +} diff --git a/crates/cyber-snapshot/src/copy/git_capture.rs b/crates/cyber-snapshot/src/copy/git_capture.rs new file mode 100644 index 000000000..cc7885def --- /dev/null +++ b/crates/cyber-snapshot/src/copy/git_capture.rs @@ -0,0 +1,62 @@ +//! Exact readonly working-tree images; never native mutation/acknowledgement authority. +use super::{links, ownership, storage}; +use cap_std::fs::Dir; +use std::collections::BTreeMap; +use std::ffi::OsStr; +use std::io::Read; +use std::os::unix::fs::MetadataExt; +use std::path::Path; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Capture { + pub policy: String, + pub files: BTreeMap, +} +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct RawImage { + pub mode: &'static str, + pub bytes: Vec, +} +pub(super) fn read(dir: &Dir, name: &OsStr) -> storage::Result { + const LIMIT: u64 = 64 * 1024 * 1024; + if dir + .symlink_metadata(name) + .map_err(storage::fail)? + .file_type() + .is_symlink() + { + return links::read(dir, name, LIMIT).map(|(_, bytes)| RawImage { + mode: "120000", + bytes, + }); + } + let mut file = storage::open(dir, Path::new(name))?; + cyber_core::binary::native::plain_snapshot_metadata(&file) + .map_err(|_| "Unsupported Git raw capture metadata")?; + let before = ownership::generation(&file)?; + let metadata = file.metadata().map_err(storage::fail)?; + if metadata.len() > LIMIT { + return Err("Git raw capture file exceeds limit".into()); + } + let mut bytes = Vec::new(); + Read::by_ref(&mut file) + .take(LIMIT + 1) + .read_to_end(&mut bytes) + .map_err(storage::fail)?; + let current = storage::open(dir, Path::new(name))?; + if bytes.len() as u64 != metadata.len() + || ownership::generation(&file)? != before + || ownership::generation(¤t)? != before + || storage::identity(¤t)? != storage::identity(&file)? + { + return Err("Git raw capture source changed while reading".into()); + } + Ok(RawImage { + mode: if metadata.mode() & 0o111 == 0 { + "100644" + } else { + "100755" + }, + bytes, + }) +} diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs index 5060e9aaf..f64101ad7 100644 --- a/crates/cyber-snapshot/src/copy/git_rules.rs +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -227,18 +227,20 @@ fn failed(error: impl std::fmt::Display) -> RestoreError { RestoreError::Failed(fail(error)) } -/// Collect only read-only policy/state inputs. This is not native effect ownership. -pub(crate) fn capture_policy( +/// Collect read-only policy and raw images. This is not native effect ownership. +pub(crate) fn capture_state( worktree: &Path, prefix: &str, inputs: &IgnoreInputs, -) -> Result { +) -> Result { let mut capture = PolicyCapture { rules: Rules::open(worktree, prefix, inputs)?, ignored: std::collections::BTreeSet::new(), marks: vec![], nodes: 0, bytes: 0, + files: std::collections::BTreeMap::new(), + content_bytes: 0, }; let mut directory = capture.rules.root.try_clone().map_err(failed)?; for part in Path::new(prefix).components() { @@ -259,7 +261,10 @@ pub(crate) fn capture_policy( } } capture.rules.verify()?; - capture.rules.fingerprint(&capture.ignored) + Ok(super::git_capture::Capture { + policy: capture.rules.fingerprint(&capture.ignored)?, + files: capture.files, + }) } struct PolicyCapture { rules: Rules, @@ -267,6 +272,8 @@ struct PolicyCapture { marks: Vec<(String, String)>, nodes: usize, bytes: usize, + files: std::collections::BTreeMap, + content_bytes: usize, } impl PolicyCapture { fn walk( @@ -330,6 +337,19 @@ impl PolicyCapture { return Err(failed("Embedded Git policy proof is unavailable")); } self.walk(&child, &relative, prefix, depth + 1)?; + } else { + if self.files.len() >= 4096 { + return Err(failed("Git raw capture file count exceeds limit")); + } + let image = super::git_capture::read(dir, &leaf).map_err(failed)?; + self.content_bytes = self + .content_bytes + .checked_add(image.bytes.len()) + .ok_or_else(|| failed("Git raw capture size overflow"))?; + if self.content_bytes > 200 * 1024 * 1024 { + return Err(failed("Git raw capture exceeds limit")); + } + self.files.insert(full, image); } } if dir_generation(dir)? != before { diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 898c66baf..ad6f806a8 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,5 +1,6 @@ //! Location-bound native recovery journals and private non-Git BLAKE3 snapshots. mod git; +pub(crate) mod git_capture; pub(crate) mod git_rules; mod inventory; mod links; diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index d521341c6..2b9dc88e1 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -12,6 +12,8 @@ pub(crate) mod history; mod images; #[cfg(any(target_os = "macos", target_os = "linux"))] pub(crate) mod prepared; +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod raw; /// A shadow git directory bound to one worktree and scoped to one Location. #[derive(Debug, Clone)] @@ -171,7 +173,7 @@ impl Shadow { let skipped = self.large_untracked(max_bytes).await; self.write_excludes(ignore, &skipped)?; #[cfg(any(target_os = "macos", target_os = "linux"))] - let policy_before = self.capture_policy(ignore, max_bytes).await; + let capture_before = self.capture_git_state(ignore, max_bytes).await; let excluded = self.omitted().await?; let cached_paths: Vec<_> = excluded .iter() @@ -191,32 +193,34 @@ impl Shadow { .to_string(); let skipped = excluded.clone(); #[cfg(any(target_os = "macos", target_os = "linux"))] - let (policy, skipped) = match policy_before { + let (policy, raw, skipped) = match capture_before { Ok(before) => { - let after = self.capture_policy(ignore, max_bytes).await?; + let after = self.capture_git_state(ignore, max_bytes).await?; if before != after { - return Err("Git policy changed during checkpoint capture".into()); + return Err("Git policy or raw state changed during checkpoint capture".into()); } - (Some(after), skipped) + let raw = self.write_raw_capture(&after.files).await?; + (Some(after.policy), Some(raw), skipped) } Err(_) => { let mut skipped = skipped; - skipped.push("native Git policy proof unavailable for this capture".into()); - (None, skipped) + skipped + .push("native Git policy/raw-state proof unavailable for this capture".into()); + (None, None, skipped) } }; #[cfg(not(any(target_os = "macos", target_os = "linux")))] - let policy = None; - let tree = self.record_snapshot(&tree, excluded, policy).await?; + let (policy, raw) = (None, None); + let tree = self.record_snapshot(&tree, excluded, policy, raw).await?; Ok(Snapshot { tree, skipped }) } #[cfg(any(target_os = "macos", target_os = "linux"))] - async fn capture_policy( + async fn capture_git_state( &self, ignore: &[String], max_untracked: u64, - ) -> Result { + ) -> Result { let mut inputs = self.ignore_inputs().await?; inputs.settings = prepared::CaptureSettings { ignore: ignore.to_vec(), @@ -229,7 +233,7 @@ impl Shadow { self.pathspec.clone() }; tokio::task::spawn_blocking(move || { - crate::copy::git_rules::capture_policy(&worktree, &prefix, &inputs) + crate::copy::git_rules::capture_state(&worktree, &prefix, &inputs) }) .await .map_err(|error| error.to_string())? @@ -302,6 +306,8 @@ impl Shadow { self.image_catalog(from).await?; self.image_catalog(to).await?; let excluded = self.excluded_between(from, to).await?; + let from = self.snapshot_metadata(from).await?.0; + let to = self.snapshot_metadata(to).await?.0; let status = split_z( &self .run(&[ @@ -309,8 +315,8 @@ impl Shadow { "--name-status", "--no-renames", "-z", - from, - to, + &from, + &to, "--", &self.pathspec, ]) @@ -323,8 +329,8 @@ impl Shadow { "--numstat", "--no-renames", "-z", - from, - to, + &from, + &to, "--", &self.pathspec, ]) @@ -354,8 +360,8 @@ impl Shadow { "--no-ext-diff", "--no-color", "--no-renames", - from, - to, + &from, + &to, "--", file, ]) @@ -386,12 +392,12 @@ impl Shadow { changes: Vec, ) -> Result { use tokio::io::AsyncWriteExt; - self.recorded_exclusions(current).await?; + let current = self.snapshot_metadata(current).await?.0; let index = PreviewIndex( self.git_dir .join(format!("preview-{}.index", ulid::Ulid::new())), ); - self.index_command(&index.0, &["read-tree", current]) + self.index_command(&index.0, &["read-tree", ¤t]) .await?; for change in changes { match change.image { diff --git a/crates/cyber-snapshot/src/git/catalog.rs b/crates/cyber-snapshot/src/git/catalog.rs index 1e61a79a8..eecee4c5e 100644 --- a/crates/cyber-snapshot/src/git/catalog.rs +++ b/crates/cyber-snapshot/src/git/catalog.rs @@ -87,7 +87,7 @@ impl Shadow { Ok(catalog) } - async fn read_catalog( + pub(super) async fn read_catalog( &self, root: String, excluded: &[String], diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index 4a849c5f9..4ca4cf1fb 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -149,7 +149,7 @@ impl Shadow { let mut trees = Vec::new(); for (index, id) in ids.into_iter().enumerate() { let mut entries = BTreeMap::new(); - for (path, (mode, object)) in self.image_catalog(id).await?.iter() { + for (path, (mode, object)) in self.native_image_catalog(id).await?.iter() { let omitted = if index == 2 { history::excludes(&expected_excluded, path) } else { diff --git a/crates/cyber-snapshot/src/git/raw.rs b/crates/cyber-snapshot/src/git/raw.rs new file mode 100644 index 000000000..19a10de03 --- /dev/null +++ b/crates/cyber-snapshot/src/git/raw.rs @@ -0,0 +1,67 @@ +//! Raw working-tree graphs stay reachable alongside the canonical tree. +use super::{Shadow, images}; +use crate::copy::git_capture::RawImage; +use std::collections::BTreeMap; + +impl Shadow { + pub(super) async fn write_raw_capture( + &self, + files: &BTreeMap, + ) -> Result { + let mut directories: BTreeMap> = BTreeMap::from([(String::new(), vec![])]); + for (path, image) in files { + images::validate_path(path)?; + let object = String::from_utf8( + self.input( + &["hash-object", "-w", "--stdin", "--no-filters"], + &image.bytes, + ) + .await?, + ) + .map_err(|_| "Invalid raw Git blob identity")? + .trim() + .to_owned(); + images::verify(&object, &image.bytes)?; + let (parent, leaf) = path.rsplit_once('/').unwrap_or(("", path)); + let mut ancestor = parent; + while !ancestor.is_empty() { + directories.entry(ancestor.into()).or_default(); + ancestor = ancestor.rsplit_once('/').map_or("", |(parent, _)| parent); + } + directories + .entry(parent.into()) + .or_default() + .extend(format!("{} blob {object}\t{leaf}\0", image.mode).bytes()); + } + let mut order: Vec<_> = directories.keys().cloned().collect(); + order.sort_by_key(|path| { + std::cmp::Reverse(path.matches('/').count() + usize::from(!path.is_empty())) + }); + for directory in order { + let bytes = directories + .remove(&directory) + .ok_or("Missing raw Git directory")?; + let object = self.write_tree_records(&bytes).await?; + if directory.is_empty() { + return Ok(object); + } + let (parent, leaf) = directory.rsplit_once('/').unwrap_or(("", &directory)); + directories + .get_mut(parent) + .ok_or("Missing raw Git parent")? + .extend(format!("040000 tree {object}\t{leaf}\0").bytes()); + } + Err("Missing raw Git root".into()) + } + pub(super) async fn write_tree_records(&self, records: &[u8]) -> Result { + let object = String::from_utf8(self.input(&["mktree", "-z"], records).await?) + .map_err(|_| "Invalid raw Git tree identity")? + .trim() + .to_owned(); + let bytes = self + .run_bounded(&["cat-file", "tree", &object], images::TREE_LIMIT) + .await?; + images::verify_object(&object, "tree", &bytes)?; + Ok(object) + } +} diff --git a/crates/cyber-snapshot/src/git_history.rs b/crates/cyber-snapshot/src/git_history.rs index ee67ad96c..bde934699 100644 --- a/crates/cyber-snapshot/src/git_history.rs +++ b/crates/cyber-snapshot/src/git_history.rs @@ -17,6 +17,8 @@ struct Record { excluded: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] policy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + raw: Option, } fn oid(id: &str) -> bool { matches!(id.len(), 40 | 64) @@ -99,6 +101,7 @@ impl Shadow { tree: &str, excluded: Vec, policy: Option, + raw: Option, ) -> Result { if !oid(tree) { return Err("Invalid snapshot tree identity".into()); @@ -111,18 +114,37 @@ impl Shadow { }) { return Err("Invalid Git policy fingerprint".into()); } + if raw + .as_ref() + .is_some_and(|raw| !oid(raw) || raw.len() != tree.len()) + || (raw.is_some() && policy.is_none()) + { + return Err("Invalid Git raw capture identity".into()); + } + let root = if let Some(raw) = &raw { + self.envelope_tree(tree, raw).await? + } else { + tree.to_owned() + }; let record = Record { - version: if policy.is_some() { 2 } else { 1 }, + version: if raw.is_some() { + 3 + } else if policy.is_some() { + 2 + } else { + 1 + }, location: self.worktree.join(&self.pathspec), tree: tree.into(), excluded, policy, + raw, }; let bytes = serde_json::to_vec(&record).map_err(|e| e.to_string())?; if bytes.len() > LIMIT { return Err("Snapshot exclusions exceed record limit".into()); } - let mut tag = format!("object {tree}\ntype tree\ntag cyber-snapshot-v{}\ntagger Cyber Snapshot 0 +0000\n\n", record.version).into_bytes(); + let mut tag = format!("object {root}\ntype tree\ntag cyber-snapshot-v{}\ntagger Cyber Snapshot 0 +0000\n\n", record.version).into_bytes(); tag.extend(bytes); let id = String::from_utf8(self.input(&["mktag"], &tag).await?) .map_err(|e| e.to_string())? @@ -137,6 +159,32 @@ impl Shadow { .await?; Ok(id) } + async fn envelope_tree(&self, tree: &str, raw: &str) -> Result { + let records = + format!("040000 tree {tree}\tcanonical\0") + &format!("040000 tree {raw}\traw\0"); + let id = String::from_utf8(self.input(&["mktree", "-z"], records.as_bytes()).await?) + .map_err(|_| "Invalid Git envelope identity")? + .trim() + .to_owned(); + if !oid(&id) { + return Err("Invalid Git envelope identity".into()); + } + Ok(id) + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub(super) async fn native_image_catalog( + &self, + id: &str, + ) -> Result { + let record = self + .read_record(id) + .await? + .ok_or("Git checkpoint lacks raw capture proof")?; + let raw = record.raw.ok_or("Git checkpoint lacks raw capture proof")?; + self.read_catalog(raw, &record.excluded).await + } + pub(super) async fn recorded_exclusions( &self, id: &str, @@ -191,9 +239,19 @@ impl Shadow { .split_once("\n\n") .ok_or("Invalid snapshot exclusion record")?; let record: Record = serde_json::from_str(body).map_err(|e| e.to_string())?; + let root = header + .lines() + .next() + .and_then(|line| line.strip_prefix("object ")) + .filter(|root| oid(root)) + .ok_or("Invalid Git snapshot root")?; if !matches!( - (record.version, record.policy.as_deref()), - (1, None) | (2, Some(_)) + ( + record.version, + record.policy.as_deref(), + record.raw.as_deref() + ), + (1, None, None) | (2, Some(_), None) | (3, Some(_), Some(_)) ) || record.policy.as_ref().is_some_and(|policy| { policy.len() != 64 || !policy @@ -205,13 +263,30 @@ impl Shadow { || record.excluded.iter().any(|p| !safe_path(p)) || !header.starts_with(&format!( "object {}\ntype tree\ntag cyber-snapshot-v{}\n", - record.tree, record.version + root, record.version )) { return Err( "Snapshot exclusion record belongs to another Location or is corrupt".into(), ); } + if let Some(raw) = &record.raw { + if !oid(raw) || raw.len() != record.tree.len() || root.len() != raw.len() { + return Err("Invalid Git raw capture identity".into()); + } + let bytes = self.run_bounded(&["cat-file", "tree", root], 1024).await?; + super::images::verify_object(root, "tree", &bytes)?; + if super::images::tree_entries(&bytes, root.len() / 2)? + != vec![ + ("canonical".into(), "040000".into(), record.tree.clone()), + ("raw".into(), "040000".into(), raw.clone()), + ] + { + return Err("Invalid Git capture envelope".into()); + } + } else if root != record.tree { + return Err("Git snapshot root does not match its record".into()); + } Ok(Some(record)) } pub(super) async fn excluded_between( @@ -266,10 +341,41 @@ mod policy_tests { let shadow = Shadow::locate(&base.join("data/snapshot"), &root).unwrap(); let checkpoint = shadow.track(&[], 2 * 1024 * 1024).await.unwrap(); let record = shadow.read_record(&checkpoint.tree).await.unwrap().unwrap(); - assert_eq!(record.version, 2); + assert_eq!(record.version, 3); + assert!(record.raw.is_some()); assert!(record.policy.is_some()); + let policy_only = shadow + .record_snapshot( + &record.tree, + record.excluded.clone(), + record.policy.clone(), + None, + ) + .await + .unwrap(); + assert_eq!( + shadow + .read_record(&policy_only) + .await + .unwrap() + .unwrap() + .version, + 2 + ); + assert!(shadow.native_image_catalog(&policy_only).await.is_err()); + // A valid tag hash cannot substitute its canonical tree for the envelope. + let mut wrong_envelope = format!("object {}\ntype tree\ntag cyber-snapshot-v3\ntagger Cyber Snapshot 0 +0000\n\n", record.tree).into_bytes(); + wrong_envelope.extend(serde_json::to_vec(&record).unwrap()); + let wrong_envelope = + String::from_utf8(shadow.input(&["mktag"], &wrong_envelope).await.unwrap()).unwrap(); + assert!( + shadow + .snapshot_metadata(wrong_envelope.trim()) + .await + .is_err() + ); let legacy = shadow - .record_snapshot(&record.tree, record.excluded.clone(), None) + .record_snapshot(&record.tree, record.excluded.clone(), None, None) .await .unwrap(); assert_eq!( @@ -290,7 +396,7 @@ mod policy_tests { ); assert!( shadow - .record_snapshot(&record.tree, vec![], Some("invalid".into())) + .record_snapshot(&record.tree, vec![], Some("invalid".into()), None) .await .is_err() ); diff --git a/crates/cyber-snapshot/tests/snapshots.rs b/crates/cyber-snapshot/tests/snapshots.rs index cccc8dbf7..262fba07c 100644 --- a/crates/cyber-snapshot/tests/snapshots.rs +++ b/crates/cyber-snapshot/tests/snapshots.rs @@ -42,6 +42,25 @@ impl Repo { } fn shadow_object(&self, revision: &str) -> PathBuf { let shadow = self.shadow_directory(); + // v3 tags peel to a retention envelope. Corrupt the canonical graph + // when the fixture names an ordinary snapshot path. + let canonical; + let revision = if let Some((snapshot, path)) = revision.split_once(':') { + let tag = Command::new("git") + .arg("--git-dir") + .arg(&shadow) + .args(["cat-file", "tag", snapshot]) + .output() + .unwrap(); + assert!(tag.status.success()); + let tag = String::from_utf8(tag.stdout).unwrap(); + let (_, body) = tag.split_once("\n\n").unwrap(); + let record: serde_json::Value = serde_json::from_str(body).unwrap(); + canonical = format!("{}:{path}", record["tree"].as_str().unwrap()); + canonical.as_str() + } else { + revision + }; let output = Command::new("git") .arg("--git-dir") .arg(&shadow) @@ -556,29 +575,7 @@ async fn corrupt_git_snapshot_blob_refuses_restore_and_preserves_current_source( let target = r.snaps.track(&r.dir()).await.unwrap().unwrap(); std::fs::write(r.root.join("a.txt"), "replacement snapshot image").unwrap(); let recorded = r.snaps.track(&r.dir()).await.unwrap().unwrap(); - let project = std::fs::read_dir(r.data.join("snapshot")) - .unwrap() - .next() - .unwrap() - .unwrap() - .path(); - let shadow = std::fs::read_dir(project) - .unwrap() - .map(|entry| entry.unwrap().path()) - .find(|path| path.join("HEAD").exists()) - .unwrap(); - let object = |snapshot: &str| { - let out = Command::new("git") - .arg("--git-dir") - .arg(&shadow) - .args(["rev-parse", &format!("{snapshot}:a.txt")]) - .output() - .unwrap(); - assert!(out.status.success()); - let id = String::from_utf8(out.stdout).unwrap(); - let id = id.trim(); - shadow.join("objects").join(&id[..2]).join(&id[2..]) - }; + let object = |snapshot: &str| r.shadow_object(&format!("{snapshot}:a.txt")); let original = object(&target.tree); let replacement = std::fs::read(object(&recorded.tree)).unwrap(); std::fs::set_permissions(&original, std::fs::Permissions::from_mode(0o600)).unwrap(); diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 4002427e1..c1085014c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -1996,3 +1996,72 @@ parent recovery tasks remain open. Final format/diff checks and SDK generation consistency passed. Pinned strict OpenSpec validation passed all 117 items; spec lint reported 0 errors/25 warnings. These results do not provide source-bound CI or complete recovery acceptance. + + +## Exact Git working-tree representation graphs (2026-11-07) + +Supported captures now collect exact eligible file bytes, executable classification +and symbolic-link target bytes before and after canonical Git capture. Source +reads use retained directories, no-follow unique regular-file opens, plain metadata +checks and source generation/identity revalidation; links reuse the pinned-inode +reader without following targets. Each collection is bounded by 4,096 files, +64 MiB per regular file and 200 MiB combined payload, plus existing namespace/rule +bounds. Before/after collections coexist for comparison; this is not a 200 MiB +process-memory claim or a completed retention cap. Changed successful collections +refuse publication; an unsupported initial collection returns a legacy checkpoint +with explicit unavailable policy/raw-state proof. + +Raw bytes are written with `hash-object --no-filters`, then literal NUL-delimited +`mktree` records build a separate worktree-relative graph. A version-3 tag points +to an envelope with exactly two typed tree links: canonical and raw. The reader +verifies the tag, envelope hash, exact links and each traversed raw directory/blob +hash. Thus ordinary Git GC retains both graphs through the existing immutable +checkpoint ref rather than relying on a raw tree ID embedded only in JSON. +Native preparation now requires raw proof for target/recorded/expected and retains +original checkpoint IDs in journals. It never promotes a bare/v1/v2 canonical +blob into raw proof. Earlier records remain readable through the public Git API. + +Public diff and projected preview-index construction explicitly resolve the +canonical tree. Existing blob/change/exclusion behavior stays canonical. The first +full suite exposed envelope use in diff and preview construction; those operations +were corrected, and corruption fixtures now deliberately resolve canonical graph +paths while still testing envelope/tag corruption. Fixtures do not weaken hash or +source/index-preservation assertions. + +Five new native/real-Git regressions cover CRLF restoration plus aggressive GC, +normalization-equivalent later owner-byte refusal, UTF-16LE representation with +SHA-256 shadow graphs, corrupt raw blob refusal, and a deliberately configured +fixture filter that changes only its temp repository during capture. The last +case verifies no new checkpoint ref is published after changed raw state. History +coverage includes v1/v2/v3 and bare reading, policy-only native refusal and a valid +forged tag whose canonical root cannot substitute for the envelope. Existing +scoped/literal paths, link images, merge/recovery, exclusions and Git source/index +preservation tests remain included. + +The SHA-256 user-repository experiment revealed public libgit2 worktree discovery +returns no root, selecting the non-Git backend. This increment does not fix that +compatibility gap. The passing graph-width fixture explicitly creates a SHA-256 +shadow over an ordinarily discovered worktree; it proves the raw graph codec, not +public SHA-256 repository discovery. Its initial failed discovery expectation is +retained here rather than characterized as accepted support. A test compile failure +from calling a private helper was fixed by invoking fixture-owned Git explicitly. + +The code-review skill was applied to all changed source/tests/artifacts. Review +checked route/type/identity validation, unknown leaf nonblocking opens, file/link +payload bounds, no-filter object writes, graph reachability, typed graph hashes, +exact envelope framing, legacy rejection and the public canonical compatibility +paths. No critical/major issue remains in this disabled raw materialization scope. +Still required: preview/path-selection parity for normalization-only changes, +configuration-resolution races and complete attribute/filter behavior, public +SHA-256 discovery, gitlinks, file/directory transitions, remaining link/large-file +compatibility, actual Git worker lifetime and durable acknowledgement, reference- +aware reclamation, performance and real client/platform acceptance. Canonical +capture still executes configured Git filters; this is not process isolation. +Ordinary Git availability/reservation remains disabled; parent tasks stay open. + +Final local macOS arm64 verification: locked snapshot tests passed 133 cases with +two ignored fixtures (the pathspec fixture is invoked by its parent). Snapshot +all-target locked Clippy passed with warnings denied. Format/diff and generated +SDK consistency passed. Pinned strict OpenSpec validation passed all 117 items; +spec lint reported 0 errors/25 warnings. Full workspace/SDK execution, Linux, +Windows, real CLI/TUI/App and source-bound CI acceptance were not run here. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 39760bc42..dfce61b34 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -541,3 +541,17 @@ fingerprint with the expected tag before journaling. This is a policy/state-inpu precondition, not worker, mutation, acknowledgement or reclamation authority. Raw working-tree representation and the remaining admission/compatibility gates remain required; a policy fingerprint alone does not activate ordinary Git recovery. + +### Exact captured Git working-tree representations + +Supported native captures SHALL bracket canonical Git capture with bounded, +no-follow reads of eligible regular files and link target bytes, refusing changed +successful captures. A version-3 tag SHALL retain a typed envelope tree linking +both the canonical Git tree and a raw representation tree. Both graphs remain +reachable from the checkpoint ref. Legacy consumers SHALL resolve the canonical +tree; native materialization SHALL use verified raw images without running clean +or smudge filters inside the filesystem worker. Historical IDs/exclusions remain +intact. Unsupported capture SHALL retain explicit unavailable proof and SHALL NOT +silently promote canonical blobs into raw-state evidence. This does not admit +ordinary Git workers or resolve their lifetime, acknowledgement, configuration +race, reclamation or complete compatibility/acceptance gates. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index b629079de..dcbe46fb0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -247,6 +247,12 @@ - **AND** successful policy collections before and after capture SHALL agree; unsupported capture SHALL report unavailable proof without inventing it, and legacy tags/bare trees SHALL remain readable without acquiring native admission authority - **AND** this policy fingerprint SHALL NOT substitute for raw working-tree state, exact worker ownership, native acknowledgement or reference-aware reclamation +#### Scenario: Canonical Git blobs differ from captured owner bytes +- **WHEN** Git normalizes a captured file's line endings or working-tree encoding +- **THEN** supported native checkpoints SHALL retain exact bounded raw file/link images alongside the canonical tree in a verified envelope graph reachable from the immutable checkpoint ref +- **AND** native expected-state validation and restoration SHALL use verified raw images, retaining original snapshot IDs and historical exclusions without invoking filters inside the filesystem worker +- **AND** legacy public Git diff/blob behavior SHALL resolve the canonical tree, older records SHALL remain readable without acquiring raw proof, and changed successful captures SHALL refuse publication + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 5b9c864dc..7eacc8f98 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -314,3 +314,14 @@ Application ignore/size settings and private roots are included. Raw working-tre representation, configuration-resolution races, Git compatibility, actual worker lifetime/acknowledgement, reclamation and full clients/platform acceptance remain open. Git admission stays disabled; M2–M8 and parent tasks remain incomplete. + + +Raw-state continuation: supported captures now bracket Git tree creation with +bounded exact raw file/link reads. Version-3 tags retain canonical and raw graphs +through a verified envelope tree; native preparation uses raw graphs, preserving +original checkpoint identities and exclusion rules. Public Git diff/blob behavior +resolves the canonical graph. Legacy bare/v1/v2 checkpoints remain readable but +cannot supply missing raw native proof. Git runtime ownership/acknowledgement, +preview/path-selection parity for normalized-only changes, configuration races, +remaining Git compatibility, retention and full client/platform acceptance still +block ordinary admission. M2–M8 and parent tasks remain incomplete. From 0dde140815f648a3832c93a33f6e67eef6c2c27c Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 04:07:58 -0400 Subject: [PATCH 26/87] Share native Git planning with managed preview and raw path selection --- crates/cyber-snapshot/src/copy/git.rs | 37 ++++++++ crates/cyber-snapshot/src/copy/git/tests.rs | 85 +++++++++++++++++++ crates/cyber-snapshot/src/git/raw.rs | 14 +++ crates/cyber-snapshot/src/lib.rs | 57 +++++++++++++ .../implementation.md | 43 ++++++++++ .../managed-recovery.md | 7 ++ .../specs/snapshots-checkpoints/spec.md | 6 ++ .../complete-daily-coding-surfaces/tasks.md | 11 +++ 8 files changed, 260 insertions(+) diff --git a/crates/cyber-snapshot/src/copy/git.rs b/crates/cyber-snapshot/src/copy/git.rs index 920b5daf3..207ab117f 100644 --- a/crates/cyber-snapshot/src/copy/git.rs +++ b/crates/cyber-snapshot/src/copy/git.rs @@ -72,6 +72,43 @@ fn matches(entry: Option<&Entry>, image: Option<&Image>) -> bool { } impl Store { + pub(crate) fn preview_git( + &self, + prepared: &PreparedRestore, + ) -> std::result::Result, RestoreError> { + let (changes, _) = self.plan_git(prepared)?; + changes + .iter() + .map(|change| { + // Current Git images were inspected without copying source bytes + // into the native CAS. Use the already verified expected packet + // for rendering, rather than reopening source or following links. + if let Some(before) = &change.before { + let image = prepared + .expected + .images + .get(&change.path) + .ok_or(RestoreError::Stale)?; + if self + .object(&image.bytes) + .map_err(RestoreError::Failed)? + .object + != before.object + { + return Err(RestoreError::Stale); + } + } + let path = if prepared.prefix.is_empty() { + change.path.clone() + } else { + format!("{}/{}", prepared.prefix, change.path) + }; + self.file_diff(&path, change.before.as_ref(), change.after.as_ref()) + .map_err(RestoreError::Failed) + }) + .collect() + } + /// Called only inside a retained synchronous worker. Ordinary Git admission /// stays disabled until full state/lifetime and native acceptance gates are ready. pub(crate) fn restore_git_worker( diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index fdf946229..e63c32e39 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -1293,3 +1293,88 @@ async fn git_capture_does_not_publish_a_checkpoint_if_a_fixture_filter_changes_r b"changed" ); } + +#[tokio::test] +async fn native_git_preview_and_path_selection_include_normalization_only_changes() { + let repo = Repo::new(); + std::fs::write(repo.root.join(".gitattributes"), b"*.txt text eol=lf\n").unwrap(); + std::fs::write(repo.root.join("file.txt"), b"same\r\n").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file.txt"), b"same\n").unwrap(); + let recorded = repo.track(&repo.root).await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + assert!(shadow.changed(&target, &recorded).await.unwrap().is_empty()); + assert_eq!( + shadow.changed_native(&target, &recorded).await.unwrap(), + ["file.txt"] + ); + assert!( + !repo + .snapshots + .owned_restore_available(repo.root.to_str().unwrap()) + ); + let preview = repo + .snapshots + .git_preview(SnapshotRestoreRequest { + directory: repo.root.to_str().unwrap().into(), + target: target.clone(), + recorded: recorded.clone(), + expected: recorded.clone(), + paths: None, + }) + .await + .unwrap(); + assert_eq!(preview.len(), 1); + assert_eq!(preview[0].file, "file.txt"); + assert!(!preview[0].patch.is_empty()); + assert_eq!( + std::fs::read(repo.root.join("file.txt")).unwrap(), + b"same\n" + ); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert!(store.recovery().unwrap().is_empty()); + assert_eq!( + store.restore_git_worker(prepared, None).unwrap(), + ["file.txt"] + ); + assert_eq!( + std::fs::read(repo.root.join("file.txt")).unwrap(), + b"same\r\n" + ); +} + +#[tokio::test] +async fn native_git_scoped_preview_matches_restore_selection_and_refuses_stale_state() { + let repo = Repo::new(); + let directory = repo.root.join("scope"); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("one"), b"before one").unwrap(); + std::fs::write(directory.join("two"), b"before two").unwrap(); + let target = repo.track(&directory).await; + std::fs::write(directory.join("one"), b"after one").unwrap(); + std::fs::write(directory.join("two"), b"after two").unwrap(); + let recorded = repo.track(&directory).await; + let request = || SnapshotRestoreRequest { + directory: directory.to_str().unwrap().into(), + target: target.clone(), + recorded: recorded.clone(), + expected: recorded.clone(), + paths: Some(vec!["scope/one".into()]), + }; + let preview = repo.snapshots.git_preview(request()).await.unwrap(); + assert_eq!(preview.len(), 1); + assert_eq!(preview[0].file, "scope/one"); + assert!(preview[0].patch.contains("a/scope/one")); + assert_eq!(std::fs::read(directory.join("one")).unwrap(), b"after one"); + assert!(repo.store(&directory).recovery().unwrap().is_empty()); + std::fs::write(directory.join("two"), b"later owner edit outside selection").unwrap(); + assert!(matches!( + repo.snapshots.git_preview(request()).await, + Err(RestoreError::Stale) + )); + assert_eq!(std::fs::read(directory.join("one")).unwrap(), b"after one"); + assert!(repo.store(&directory).recovery().unwrap().is_empty()); +} diff --git a/crates/cyber-snapshot/src/git/raw.rs b/crates/cyber-snapshot/src/git/raw.rs index 19a10de03..7e389199d 100644 --- a/crates/cyber-snapshot/src/git/raw.rs +++ b/crates/cyber-snapshot/src/git/raw.rs @@ -4,6 +4,20 @@ use crate::copy::git_capture::RawImage; use std::collections::BTreeMap; impl Shadow { + pub(crate) async fn changed_native(&self, from: &str, to: &str) -> Result, String> { + let excluded = self.excluded_between(from, to).await?; + let before = self.native_image_catalog(from).await?; + let after = self.native_image_catalog(to).await?; + let names: std::collections::BTreeSet<_> = + before.keys().chain(after.keys()).cloned().collect(); + Ok(names + .into_iter() + .filter(|path| { + before.get(path) != after.get(path) && !super::history::excludes(&excluded, path) + }) + .collect()) + } + pub(super) async fn write_raw_capture( &self, files: &BTreeMap, diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 9aab51824..c00f75543 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -164,6 +164,35 @@ impl GitSnapshots { } } + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn git_preview( + &self, + request: cyber_server::runtime::SnapshotRestoreRequest, + ) -> Result, RestoreError> { + self.recovery_location(&request.directory) + .map_err(RestoreError::Failed)?; + let shadow = self + .shadow(&request.directory) + .ok_or_else(|| RestoreError::Failed("Git snapshots are unavailable".into()))?; + let options = self.copy_options(&request.directory); + let lock = cyber_server::workspace::mutation_lock(&shadow.worktree) + .map_err(|error| RestoreError::Failed(error.to_string()))?; + let guard = lock.lock_owned().await; + let prepared = shadow + .prepare_restore(&request, options.max_file, options.max_total) + .await + .map_err(RestoreError::Failed)?; + let data = self.data_dir.clone(); + tokio::task::spawn_blocking(move || { + let _guard = guard; + let store = copy::Store::open(&data, &request.directory, options) + .map_err(RestoreError::Failed)?; + store.preview_git(&prepared) + }) + .await + .map_err(|error| RestoreError::Failed(error.to_string()))? + } + fn options(&self, location: &Path) -> (Vec, u64) { Self::options_value(&(self.config)(location)) } @@ -603,6 +632,10 @@ impl Snapshots for GitSnapshots { let Some(shadow) = self.shadow(&directory) else { return Ok(Vec::new()); }; + #[cfg(any(target_os = "macos", target_os = "linux"))] + if self.owned_restore_available(&directory) { + return shadow.changed_native(&from, &to).await; + } shadow.changed(&from, &to).await }) } @@ -680,6 +713,18 @@ impl Snapshots for GitSnapshots { "Non-Git restore is unavailable on this host".into(), )); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + if self.owned_restore_available(&directory) { + return self + .git_preview(cyber_server::runtime::SnapshotRestoreRequest { + directory, + target, + recorded, + expected, + paths: None, + }) + .await; + } let shadow = self .shadow(&directory) .ok_or_else(|| RestoreError::Failed("Snapshots are unavailable".into()))?; @@ -782,6 +827,18 @@ impl Snapshots for GitSnapshots { "Non-Git restore is unavailable on this host".into(), )); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + if self.owned_restore_available(&directory) { + return self + .git_preview(cyber_server::runtime::SnapshotRestoreRequest { + directory, + target, + recorded, + expected, + paths: Some(paths), + }) + .await; + } let shadow = self .shadow(&directory) .ok_or_else(|| RestoreError::Failed("Snapshots are unavailable".into()))?; diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index c1085014c..e30d19d82 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2065,3 +2065,46 @@ all-target locked Clippy passed with warnings denied. Format/diff and generated SDK consistency passed. Pinned strict OpenSpec validation passed all 117 items; spec lint reported 0 errors/25 warnings. Full workspace/SDK execution, Linux, Windows, real CLI/TUI/App and source-bound CI acceptance were not run here. + + +## Native Git preview and raw path selection (2026-11-07) + +`Store::preview_git` now calls the same native raw-image planner as restore, with +full expected namespace/policy/image validation, historical mutation exclusions, +selected paths, merge/conflict behavior and pre-effect directory checks. Rendering +uses the verified expected packet for before images: inspected Git source bytes +are not necessarily in the native CAS. This was caught during review and corrected +without reopening source or following links. Preview may populate private CAS +objects but publishes no journal or source effect. Git file paths and unified diff +headers are worktree-relative, including subdirectory Locations. + +The GitSnapshots preview and selected-preview ports use this implementation only +when ordinary owned Git is admitted. `changed` similarly selects verified raw +catalogs behind that same gate, retaining normalization-only path differences. +The actual gate remains closed for Git; legacy public behavior remains unchanged. +An async adapter retains the worktree mutation lock across bounded immutable +preparation and the synchronous native preview callback. No Git subprocess runs +inside that callback. No preview-only activation or worker authority is added. + +Two backend/adapter fixtures exercise normalization-only CRLF differences, raw +changed-path selection versus canonical compatibility, preview/apply path parity, +source/journal preservation, scoped selected paths/patch headers and stale owner +edits outside selection. The fixtures explicitly verify ordinary Git admission +remains disabled. These are not real CLI/TUI/App or full runtime acceptance. + +The code-review skill was applied to the complete source/test/artifact diff, +including expected-packet rendering, source versus private CAS effects, scoped +path translation, lock retention, full-state versus selected-mutation checks and +the unchanged admission gate. No critical/major finding remains in this disabled +preview scope. Raw summary/client parity, complete worker lifetime/checkout +acknowledgement, reclamation, Git compatibility and platform/crash acceptance still +block ordinary admission; all parent tasks remain open. + +Final local macOS arm64 verification: locked snapshot tests passed 135 cases with +two ignored fixtures (the pathspec fixture is invoked by its parent). Snapshot +all-target locked Clippy passed with warnings denied. Format/diff and generated +SDK consistency passed. Pinned strict OpenSpec passed all 117 items; spec lint +reported 0 errors/25 warnings. An initial fixture compile error assumed the request +was Clone; the test now constructs independent identical requests without changing +the production API. Full workspace/SDK execution, real CLI/TUI/App, Linux/Windows +and source-bound CI acceptance were not run for this increment. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index dfce61b34..76eebddc8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -555,3 +555,10 @@ intact. Unsupported capture SHALL retain explicit unavailable proof and SHALL NO silently promote canonical blobs into raw-state evidence. This does not admit ordinary Git workers or resolve their lifetime, acknowledgement, configuration race, reclamation or complete compatibility/acceptance gates. + +Managed Git preview SHALL use the same native raw-image planner as restore, +including expected-state/policy revalidation, historical exclusions, allowed paths +and all-plan conflict refusal. Preview SHALL publish no journal or source effects. +Its client path SHALL remain behind the ordinary owned-Git availability gate; +connecting preview alone SHALL NOT enable native source admission. Raw changed-path +selection SHALL preserve normalization-only changes when managed Git is admitted. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index dcbe46fb0..9f369d0f3 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -253,6 +253,12 @@ - **AND** native expected-state validation and restoration SHALL use verified raw images, retaining original snapshot IDs and historical exclusions without invoking filters inside the filesystem worker - **AND** legacy public Git diff/blob behavior SHALL resolve the canonical tree, older records SHALL remain readable without acquiring raw proof, and changed successful captures SHALL refuse publication +#### Scenario: Managed Git preview and selected paths use the native plan +- **WHEN** an admitted managed Git restore previews a checkpoint whose raw bytes differ only by normalization +- **THEN** changed-path selection and preview SHALL preserve that difference and preview SHALL use the same raw-image planner, expected-state/policy checks, allowed paths and whole-plan conflicts as native restore +- **AND** Git paths and patch headers SHALL remain worktree-relative, including scoped Locations, and preview SHALL publish no journal or source mutation +- **AND** connecting this path SHALL NOT enable ordinary owned Git admission before its remaining gates are satisfied + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 7eacc8f98..a4d6bc11e 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -325,3 +325,14 @@ cannot supply missing raw native proof. Git runtime ownership/acknowledgement, preview/path-selection parity for normalized-only changes, configuration races, remaining Git compatibility, retention and full client/platform acceptance still block ordinary admission. M2–M8 and parent tasks remain incomplete. + + +Native preview continuation: managed Git preview is connected to the same raw +planner as restore behind the unchanged owned-Git availability gate. Raw changed- +path selection is connected at the same gate, preserving normalization-only edits. +Rendering uses verified expected images rather than reopening current source, and +scoped Git paths/patch headers stay worktree-relative. Backend fixtures verify +preview leaves no journal/source changes and revalidates outside selected paths. +Complete summaries/client behavior, worker lifetime/acknowledgement, reclamation, +remaining Git compatibility and platform acceptance remain required. Ordinary Git +admission and all M2–M8/parent task acceptance remain unchanged and incomplete. From 99aa6f81e688868be8a1b091616c330e2869a5ac Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 04:11:42 -0400 Subject: [PATCH 27/87] Reserve exact native snapshot lifetimes for Git Locations --- crates/cyber-snapshot/src/copy/git/tests.rs | 92 +++++++++++++++++++ crates/cyber-snapshot/src/lib.rs | 7 +- .../implementation.md | 34 +++++++ .../managed-recovery.md | 8 ++ .../specs/snapshots-checkpoints/spec.md | 6 ++ .../complete-daily-coding-surfaces/tasks.md | 9 ++ 6 files changed, 152 insertions(+), 4 deletions(-) diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index e63c32e39..db239abbc 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -1378,3 +1378,95 @@ async fn native_git_scoped_preview_matches_restore_selection_and_refuses_stale_s assert_eq!(std::fs::read(directory.join("one")).unwrap(), b"after one"); assert!(repo.store(&directory).recovery().unwrap().is_empty()); } + +#[tokio::test] +async fn git_lifetime_port_retains_exact_owner_lock_without_enabling_source_admission() { + use cyber_server::runtime::SnapshotLifetimeRequest; + let repo = Repo::new(); + let request = SnapshotLifetimeRequest { + activity_id: "op_git_worker".into(), + session_id: "ses_git_owner".into(), + session_created_id: "evt_git_created".into(), + directory: repo.root.clone(), + worktree_id: None, + }; + let lifetime = repo + .snapshots + .reserve_worker(request.clone()) + .await + .unwrap() + .unwrap(); + let binding = lifetime.binding().clone(); + assert_eq!(binding.version, 2); + assert_eq!(binding.owner, request); + lifetime.verify().unwrap(); + lifetime.verify_preparation(None).unwrap(); + assert_eq!(lifetime.checkout_bindings().unwrap(), None); + assert!( + repo.snapshots + .reserve_worker(request.clone()) + .await + .is_err() + ); + assert!(repo.snapshots.lock_worker(binding.clone()).await.is_err()); + assert!( + !repo + .snapshots + .owned_restore_available(repo.root.to_str().unwrap()) + ); + drop(lifetime); + let lifetime = repo.snapshots.lock_worker(binding.clone()).await.unwrap(); + assert_eq!(lifetime.binding(), &binding); + lifetime.bind_checkouts(&[]).unwrap(); + assert_eq!(lifetime.checkout_bindings().unwrap(), Some(vec![])); + drop(lifetime); + let mut foreign = binding.clone(); + foreign.owner.session_created_id = "evt_other_creation".into(); + assert!(repo.snapshots.lock_worker(foreign).await.is_err()); + let lifetime = repo + .snapshots + .reserve_worker(request) + .await + .unwrap() + .unwrap(); + assert_eq!(lifetime.binding(), &binding); + assert_eq!(lifetime.checkout_bindings().unwrap(), Some(vec![])); + assert!(repo.store(&repo.root).recovery().unwrap().is_empty()); + assert!( + !repo + .snapshots + .owned_restore_available(repo.root.to_str().unwrap()) + ); +} + +#[tokio::test] +async fn git_lifetime_port_keeps_scoped_locations_and_managed_checkout_completeness() { + use cyber_server::runtime::SnapshotLifetimeRequest; + let repo = Repo::new(); + let scope = repo.root.join("scope"); + std::fs::create_dir(&scope).unwrap(); + std::fs::write(scope.join("owner"), b"owner bytes").unwrap(); + let request = SnapshotLifetimeRequest { + activity_id: "op_git_scoped".into(), + session_id: "ses_git_owner".into(), + session_created_id: "evt_git_created".into(), + directory: scope.clone(), + worktree_id: Some("wt_required".into()), + }; + let lifetime = repo + .snapshots + .reserve_worker(request) + .await + .unwrap() + .unwrap(); + assert_eq!(lifetime.binding().owner.directory, scope); + assert!(lifetime.bind_checkouts(&[]).is_err()); + assert_eq!(lifetime.checkout_bindings().unwrap(), None); + assert!(repo.store(&scope).recovery().unwrap().is_empty()); + assert_eq!(std::fs::read(scope.join("owner")).unwrap(), b"owner bytes"); + assert!( + !repo + .snapshots + .owned_restore_available(scope.to_str().unwrap()) + ); +} diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index c00f75543..83542ff5d 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -328,10 +328,9 @@ impl Snapshots for GitSnapshots { .directory .to_str() .ok_or("Invalid snapshot lifetime Location")?; - // Git journaling and its lifetime proof are integrated separately. - if cyber_core::project::worktree_root(Path::new(directory)).is_some() { - return Ok(None); - } + // Reservation is metadata-only and independent of the snapshot + // image backend. Ordinary Git source admission remains gated by + // owned_restore_available, not by weakening lifetime ownership. self.recovery_location(directory)?; let data = self.data_dir.clone(); let options = self.recovery_options(directory); diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index e30d19d82..5eb80f8af 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2108,3 +2108,37 @@ reported 0 errors/25 warnings. An initial fixture compile error assumed the requ was Clone; the test now constructs independent identical requests without changing the production API. Full workspace/SDK execution, real CLI/TUI/App, Linux/Windows and source-bound CI acceptance were not run for this increment. + + +## Git lifetime metadata reservation (2026-11-07) + +The public native lifetime port no longer refuses Git Locations before reaching +the backend-independent version-2 lifetime engine. Reservations still canonicalize +and protect the Location, bind exact activity/Session creation/worktree ownership, +publish complete private native records and retain the actual file lock. The +ordinary owned Git availability check and restore-owned direct-call gate remain +unchanged and closed. Metadata reservation does not create an intent or source +admission, acknowledge a journal or native checkout, or authorize reclamation. + +Two real-Git port regressions verify exact retained lifetime ownership, live +reserve/reacquire refusal, exited reacquisition, durable empty complete checkout +sets for an unmanaged root, foreign Session-creation refusal, scoped Locations and +refusal to bind an empty set for a requested managed worktree. They preserve owner +files, assert no journal was published, and explicitly check source availability +remains false. These port tests do not create an actual managed checkout, runtime +Session/source callback or consuming Store receipt; full Git runtime integration +and post-effect native checkout acknowledgement remain required. + +The code-review skill was applied to the source/test/spec diff, focusing on +reservation versus source authority, canonical/private Location validation, +unchanged native owner checks and separate admission gates. No critical/major +finding remains in this metadata-only reservation scope. All parent recovery +and acceptance tasks remain open. + +Local macOS arm64 verification: locked snapshot tests passed 137 cases with two +ignored fixtures (the pathspec fixture is invoked by its parent). Snapshot +all-target locked Clippy passed with warnings denied. Format/diff and generated +SDK consistency passed. Pinned strict OpenSpec passed all 117 items; spec lint +reported 0 errors/25 warnings. Full workspace/SDK execution, actual Git runtime +source/receipt and CLI/TUI/App acceptance, Linux/Windows and source-bound CI were +not run for this increment. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 76eebddc8..2f5e3953f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -562,3 +562,11 @@ and all-plan conflict refusal. Preview SHALL publish no journal or source effect Its client path SHALL remain behind the ordinary owned-Git availability gate; connecting preview alone SHALL NOT enable native source admission. Raw changed-path selection SHALL preserve normalization-only changes when managed Git is admitted. + +Git Locations MAY reserve the same exact version-2 native lifetime metadata as +file-copy Locations before resource acquisition. Reservation SHALL validate the +canonical Location, protect private roots, retain the live inode lock and refuse +foreign owner/creation context. Reacquisition SHALL read the exact original record; +a requested managed worktree SHALL NOT accept an empty checkout set. Reservation +alone SHALL NOT grant source admission, journal acknowledgement, checkout settlement +or reclamation authority. Ordinary owned Git availability remains separately gated. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 9f369d0f3..99703be69 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -259,6 +259,12 @@ - **AND** Git paths and patch headers SHALL remain worktree-relative, including scoped Locations, and preview SHALL publish no journal or source mutation - **AND** connecting this path SHALL NOT enable ordinary owned Git admission before its remaining gates are satisfied +#### Scenario: Git lifetime reservation precedes source admission +- **WHEN** a trusted runtime reserves a native snapshot worker lifetime in a Git Location +- **THEN** the metadata port SHALL bind the exact canonical Location, Session creation and activity to a retained version-2 native inode lock and SHALL refuse live-lock or foreign-owner reacquisition +- **AND** a managed worktree SHALL require its complete checkout set, while reservation alone SHALL NOT admit source effects, acknowledge a journal or settle a checkout +- **AND** ordinary owned Git availability SHALL remain separately gated until full recovery acceptance + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index a4d6bc11e..c1d7d0079 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -336,3 +336,12 @@ preview leaves no journal/source changes and revalidates outside selected paths. Complete summaries/client behavior, worker lifetime/acknowledgement, reclamation, remaining Git compatibility and platform acceptance remain required. Ordinary Git admission and all M2–M8/parent task acceptance remain unchanged and incomplete. + + +Git lifetime-port continuation: trusted callers may now reserve backend-independent +version-2 native lifetime metadata in Git Locations. Exact live locks, owner/Session +creation/Location checks and managed-checkout completeness use the shared native +engine. This lifts only the metadata reservation refusal; ordinary owned Git +source availability remains false. Actual Git worker/checkout consuming receipts, +post-effect acknowledgement/reconciliation, reclamation and complete crash/client/ +platform acceptance remain required; M2–M8 and parent tasks remain open. From 592b6a1a7263a467c86d0170eef2317c8cc121f1 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 04:22:08 -0400 Subject: [PATCH 28/87] Bind ordinary restore receipts to exact native checkout resources --- crates/cyber-server/src/runtime/activity.rs | 21 +++ crates/cyber-server/src/runtime/mod.rs | 3 +- .../src/runtime/snapshot_restore_receipt.rs | 72 ++++++++ .../cyber-server/src/runtime/snapshot_work.rs | 32 ++++ .../src/runtime/snapshot_work_tests.rs | 142 +++++++++++++- .../src/copy/resolution_tests.rs | 1 + .../implementation.md | 56 ++++++ .../managed-recovery.md | 8 + .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 10 + sdk/openapi.json | 173 +++++++++++------- sdk/typescript/src/generated/types.ts | 8 + 12 files changed, 460 insertions(+), 72 deletions(-) diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 03ab85846..5534d3310 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -368,9 +368,15 @@ impl Scope { digest, sequence, &events, + self.snapshot_lifetime(), + &location.bindings().map_err(RuntimeError::Conflict)?, ) }) .transpose()?; + let checkout = settled + .snapshot_restore_receipt + .as_ref() + .and_then(|receipt| receipt.checkout.clone()); let bindings = authority.bindings.clone(); let acknowledgement = change(&settled); let owners = [ @@ -384,6 +390,21 @@ impl Scope { location .verify() .map_err(cyber_store::StoreError::InvalidAppend)?; + if let Some(checkout) = &checkout { + let resources = location + .bindings() + .map_err(cyber_store::StoreError::InvalidAppend)?; + if checkout + .lifetime + .checkout_resources_digest(&resources) + .map_err(cyber_store::StoreError::InvalidAppend)? + != checkout.resources_digest + { + return Err(cyber_store::StoreError::InvalidAppend( + "Snapshot consuming checkout resources changed".into(), + )); + } + } if stop.is_cancelled() { return Err(cyber_store::StoreError::InvalidAppend( "Snapshot consuming acknowledgement was cancelled".into(), diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 3b1385486..21bb563e3 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -150,7 +150,8 @@ pub use snapshot_recovery_types::{ }; pub use snapshot_work::{ SnapshotBinding, SnapshotCompletion, SnapshotNativeGuard, SnapshotNativeResult, SnapshotOrigin, - SnapshotRestoreReceipt, SnapshotRestoreRequest, SnapshotWork, SnapshotWorker, + SnapshotRestoreCheckout, SnapshotRestoreReceipt, SnapshotRestoreRequest, SnapshotWork, + SnapshotWorker, }; mod routing; mod sandbox_events; diff --git a/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs index c7af8f1de..5f6650f16 100644 --- a/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs +++ b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs @@ -5,6 +5,57 @@ use rusqlite::Connection; const MAX_EVENTS: i64 = 4096; +impl SnapshotRestoreReceipt { + /// Bind native disposition metadata to an already observed durable receipt. + /// This digest alone grants no claim acquisition, acknowledgement or replay authority. + pub fn checkout_acknowledgement_digest(&self) -> Result { + use sha2::{Digest, Sha256}; + if self.session_before < 0 + || self.session_after < self.session_before + || self.session_after - self.session_before > MAX_EVENTS + || [&self.intent_digest, &self.events_digest] + .iter() + .any(|hash| hash.len() != 64 || !hash.bytes().all(|byte| byte.is_ascii_hexdigit())) + { + return Err("Invalid restore receipt acknowledgement frame".into()); + } + let checkout = self + .checkout + .as_ref() + .ok_or("Legacy restore receipt lacks checkout proof")?; + checkout.validate(&self.origin)?; + let bytes = serde_json::to_vec(&("snapshot-restore-checkout-ack-v1", self)) + .map_err(|error| error.to_string())?; + Ok(format!("{:x}", Sha256::digest(bytes))) + } +} + +impl super::SnapshotRestoreCheckout { + pub(super) fn validate(&self, origin: &SnapshotOrigin) -> Result<(), String> { + let owner = &self.lifetime.owner; + if self.version != 1 + || self.lifetime.version != 2 + || self.lifetime.source_phase.is_none() + || origin.version != 1 + || owner.activity_id != origin.activity_id + || owner.session_id != origin.session_id + || owner.session_created_id != origin.session_created_id + || owner.worktree_id != origin.worktree_id + || owner.directory != origin.binding.directory + || self.lifetime.root_identity != origin.binding.root_identity + || self.lifetime.namespace_identity != origin.binding.namespace_identity + || self.resources_digest.len() != 64 + || !self + .resources_digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err("Restore receipt checkout ownership differs".into()); + } + self.lifetime.checkout_owner().map(drop) + } +} + impl Runtime { /// Lookup only; legacy settlements without an immutable receipt return None. pub async fn snapshot_restore_lookup( @@ -76,6 +127,8 @@ pub(super) fn capture( digest: &str, sequence: i64, events: &[NewEvent], + lifetime: Option<&super::SnapshotLifetimeBinding>, + resources: &[cyber_core::worktrees::CheckoutBinding], ) -> Result { let origin = origin.ok_or_else(|| { RuntimeError::Conflict("Immutable snapshot receipt lacks its native origin".into()) @@ -91,6 +144,18 @@ pub(super) fn capture( "Invalid immutable snapshot receipt".into(), )); } + let checkout = lifetime + .map(|lifetime| { + let checkout = super::SnapshotRestoreCheckout { + version: 1, + lifetime: lifetime.clone(), + resources_digest: lifetime.checkout_resources_digest(resources)?, + }; + checkout.validate(origin)?; + Ok::<_, String>(Box::new(checkout)) + }) + .transpose() + .map_err(RuntimeError::Conflict)?; Ok(SnapshotRestoreReceipt { origin: origin.clone(), intent_digest: digest.to_owned(), @@ -100,6 +165,7 @@ pub(super) fn capture( .ok_or_else(|| RuntimeError::Invalid("Snapshot Session sequence overflow".into()))?, events_digest: snapshot_work::restore_events_digest(events) .map_err(RuntimeError::Invalid)?, + checkout, }) } @@ -107,6 +173,12 @@ pub(super) fn capture( pub(super) fn verify(db: &Connection, receipt: &SnapshotRestoreReceipt) -> Result<(), StoreError> { let (_, native) = super::snapshot_recovery_record::attest(db, &receipt.origin)?; let invalid = || StoreError::InvalidAppend("Ordinary restore settlement proof differs".into()); + if let Some(checkout) = &receipt.checkout { + checkout.validate(&receipt.origin).map_err(|_| invalid())?; + if native.snapshot_lifetime.as_ref() != Some(&checkout.lifetime) { + return Err(invalid()); + } + } if native.status != "settled" || native.phase != "snapshot_restore_committed" || native.snapshot_restore_receipt.as_ref() != Some(receipt) diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index aec68b8e3..60d4d1587 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -43,6 +43,17 @@ pub struct SnapshotRestoreReceipt { pub session_before: i64, pub session_after: i64, pub events_digest: String, + /// Missing on historical receipts; absence never authorizes checkout acknowledgement. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkout: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotRestoreCheckout { + pub version: u32, + pub lifetime: super::SnapshotLifetimeBinding, + pub resources_digest: String, } pub(super) fn restore_events_digest(events: &[NewEvent]) -> Result { @@ -382,6 +393,27 @@ impl SnapshotCompletion { .guard .verify(self.worker.scope.snapshot_origin()) .map_err(RuntimeError::Conflict)?; + if let Some(lifetime) = &self.worker._lifetime { + lifetime.verify().map_err(RuntimeError::Conflict)?; + let published = lifetime + .checkout_bindings() + .map_err(RuntimeError::Conflict)? + .ok_or_else(|| { + RuntimeError::Conflict("Restore checkout publication is missing".into()) + })?; + if self + .worker + .location() + .map_err(RuntimeError::Conflict)? + .bindings() + .map_err(RuntimeError::Conflict)? + != published + { + return Err(RuntimeError::Conflict( + "Restore checkout publication differs".into(), + )); + } + } let location = self .worker ._location diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index d9a465b6c..276a94344 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -690,7 +690,7 @@ impl Snapshots for Lifetimes { version: 2, owner: request, root_identity: "instrumented_root".into(), - namespace_identity: "fixture-namespace".into(), + namespace_identity: "instrumented_namespace".into(), directory_identity: "fixture-directory".into(), file_identity: "fixture-file".into(), source_phase: Some("phase_fixture".into()), @@ -2995,3 +2995,143 @@ async fn rejected_runtime_commit_keeps_origin_unknown_and_never_acknowledges_the ); restarted.shutdown().await; } + +struct LostRestoreResources(AtomicUsize); +impl super::super::SnapshotLocationGuard for LostRestoreResources { + fn verify(&self) -> Result<(), String> { + Ok(()) + } + fn bindings(&self) -> Result, String> { + if self.0.fetch_add(1, Ordering::SeqCst) < 2 { + Ok(vec![]) + } else { + Err("fixture lost the complete retained checkout set".into()) + } + } +} + +#[tokio::test] +async fn ordinary_restore_receipt_binds_native_lifetime_and_rechecks_resources_at_writer() { + for case in ["valid", "writer", "publication"] { + let lost = case != "valid"; + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let resources = Arc::new(std::sync::Mutex::new(None)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + reject_resources: false, + resources: resources.clone(), + }), + ); + let mut owner = worker(&runtime, &session).await; + let id = owner.scope.id().to_owned(); + let lifetime = owner.scope.snapshot_lifetime().unwrap().clone(); + if case == "writer" { + owner._location = Some(Box::new(LostRestoreResources(AtomicUsize::new(0)))); + } + let native_binding = binding(temp.path().canonicalize().unwrap()); + let acks = Arc::new(AtomicUsize::new(0)); + let seen = acks.clone(); + let completed = owner + .run(move |work| { + let origin = work.begin(native_binding).unwrap(); + work.effect(|| Ok(())).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: seen, + }), + )) + }) + .await + .unwrap(); + if case == "publication" { + *resources.lock().unwrap() = None; + } + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + let before = state.last_seq; + let result = completed.commit( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "checkout bound".into(), + usage: None, + cost: None, + model: None, + }, + )], + ); + if lost { + assert!(result.is_err()); + assert_eq!(state.last_seq, before); + assert_ne!(state.info.title, "checkout bound"); + assert_eq!(acks.load(Ordering::SeqCst), 0); + drop(state); + assert!( + runtime + .snapshot_restore_lookup(&session, &id) + .await + .unwrap() + .is_none() + ); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + } else { + result.unwrap(); + assert_eq!(state.info.title, "checkout bound"); + drop(state); + let receipt = runtime + .snapshot_restore_lookup(&session, &id) + .await + .unwrap() + .unwrap(); + let checkout = receipt.checkout.as_ref().unwrap(); + assert_eq!(checkout.lifetime, lifetime); + assert_eq!( + checkout.resources_digest, + lifetime.checkout_resources_digest(&[]).unwrap() + ); + let hash = receipt.checkout_acknowledgement_digest().unwrap(); + let mut invalid = receipt.clone(); + invalid.events_digest = "invalid".into(); + assert!(invalid.checkout_acknowledgement_digest().is_err()); + let mut invalid = receipt.clone(); + invalid.session_after = invalid.session_before - 1; + assert!(invalid.checkout_acknowledgement_digest().is_err()); + let mut changed = receipt.clone(); + changed + .checkout + .as_mut() + .unwrap() + .lifetime + .owner + .session_created_id = "evt_foreign".into(); + assert!(changed.checkout_acknowledgement_digest().is_err()); + let mut changed = receipt.clone(); + changed.checkout.as_mut().unwrap().resources_digest = "f".repeat(64); + assert_ne!(changed.checkout_acknowledgement_digest().unwrap(), hash); + assert!( + runtime + .inner + .store + .read(move |db| super::super::snapshot_restore_receipt::verify(db, &changed)) + .is_err() + ); + let mut legacy = receipt; + legacy.checkout = None; + let bytes = serde_json::to_value(&legacy).unwrap(); + assert!(bytes.get("checkout").is_none()); + let legacy: SnapshotRestoreReceipt = serde_json::from_value(bytes).unwrap(); + assert!(legacy.checkout_acknowledgement_digest().is_err()); + assert_eq!(acks.load(Ordering::SeqCst), 1); + } + runtime.shutdown().await; + } +} diff --git a/crates/cyber-snapshot/src/copy/resolution_tests.rs b/crates/cyber-snapshot/src/copy/resolution_tests.rs index cf17ff093..5116ae1b5 100644 --- a/crates/cyber-snapshot/src/copy/resolution_tests.rs +++ b/crates/cyber-snapshot/src/copy/resolution_tests.rs @@ -76,6 +76,7 @@ fn ordinary_reconciliation_requires_completed_immutable_intent_and_current_autho session_before: 0, session_after: 1, events_digest: "a".repeat(64), + checkout: None, }; assert!( p.store diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 5eb80f8af..8c057e914 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2142,3 +2142,59 @@ SDK consistency passed. Pinned strict OpenSpec passed all 117 items; spec lint reported 0 errors/25 warnings. Full workspace/SDK execution, actual Git runtime source/receipt and CLI/TUI/App acceptance, Linux/Windows and source-bound CI were not run for this increment. + + +## Ordinary restore checkout evidence in consuming receipts (2026-11-07) + +New ordinary restore receipts optionally carry a version-1 checkout section with +the exact version-2 native lifetime and complete retained checkout-set digest. The +section is captured from the worker's original lifetime and native Location proof, +with exact activity/Session creation/worktree/Location/root/namespace checks. +The completion first verifies the retained lifetime and its original published +checkout set. The paired writer reads the retained resource set again before +admitting the Session/native append. Lost, changed or incomplete proof refuses that pair before +native acknowledgement. Lookup compares the receipt's lifetime with attested +original native history, alongside existing exact journal and Session-batch proof. + +A versioned SHA-256 acknowledgement frame covers the complete receipt, validates +its checkout owner and basic batch/digest bounds, and refuses historical receipts +without checkout evidence. The optional field is omitted for legacy receipts; +reading or hashing older receipts never invents resource proof. Existing journal +reconciliation behavior is unchanged: this increment does not publish post-effect +checkout acknowledgements or authorize cleanup. Native lifetime/checkout pins and +consuming receipts still need full actual-Git runtime and reconciliation acceptance. + +A new runtime test uses instrumented lifetime/Location guards to commit and look up +the exact checkout evidence, to lose the original resource publication before +commit, and to lose the complete resource proof between initial capture and the +paired writer. The refusal leaves Session state unchanged, records +unknown native outcome, performs no native acknowledgement and publishes no restore +receipt. Positive assertions cover original lifetime/digest identity, invalid batch +or hash frames, foreign Session creation, changed resource digest and legacy wire +omission/no hash authority. These guards are instrumented; no real managed checkout +or actual Git source callback is claimed by this test. An existing fixture namespace +was aligned with the native origin so the new exact namespace comparison remains +meaningful. The backend's manually constructed legacy receipt explicitly uses None. + +The code-review skill was applied to source, tests, schema artifacts and spec deltas, +focusing on additive historical encoding, complete-set capture versus writer +revalidation, original native-history attestation, no source/Store reentry from +native verification, bounded hash frames and preservation of ordinary admission +and unknown fences. This receipt evidence is not complete native acknowledgement +acceptance. All parent recovery and delivery tasks remain open. + +Final local macOS arm64 verification: locked server library tests passed 430 cases +with one ignored fixture; locked snapshot tests passed 137 cases with two ignored +fixtures (the pathspec fixture is invoked by its parent). Generated OpenAPI was +regenerated and its current-document test passed. SDK tests passed 134 cases with +10 CYBER_BIN-dependent end-to-end skips. Three-package all-target locked Clippy +(server/snapshot/tools) passed with Rust warnings denied; native C compilation +reported sandbox_init/sandbox_free_error deprecation warnings. Their origin was +not compared with main, and no pre-existing attribution is made. An earlier +Clippy run caught the enlarged acknowledgement enum; boxing the optional checkout +section corrected its in-memory size while preserving JSON shape. Final format/ +diff and SDK generation consistency passed. Strict pinned OpenSpec passed all +117 items; spec lint reported 0 errors/25 warnings. Real Git source/managed-checkout +publication, CLI/TUI/App, Linux/Windows, full workspace and source-bound CI +acceptance were not run here. No post-effect native checkout marker is published +by this increment and no parent recovery task is accepted. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 2f5e3953f..70a2ad621 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -570,3 +570,11 @@ foreign owner/creation context. Reacquisition SHALL read the exact original reco a requested managed worktree SHALL NOT accept an empty checkout set. Reservation alone SHALL NOT grant source admission, journal acknowledgement, checkout settlement or reclamation authority. Ordinary owned Git availability remains separately gated. + +New ordinary restore consuming receipts SHALL bind the exact version-2 lifetime +and a versioned digest of the complete retained checkout set when native lifetime +proof exists. The paired writer SHALL revalidate that complete set before commit; +lookup SHALL verify the receipt's lifetime against the original native history. +Historical receipts without this field remain readable without gaining checkout +acknowledgement authority. The receipt field is evidence for subsequent exact +native acknowledgement, not publication or reclamation authority by itself. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 99703be69..2df161a04 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -265,6 +265,12 @@ - **AND** a managed worktree SHALL require its complete checkout set, while reservation alone SHALL NOT admit source effects, acknowledge a journal or settle a checkout - **AND** ordinary owned Git availability SHALL remain separately gated until full recovery acceptance +#### Scenario: Ordinary restore receipt retains complete checkout evidence +- **WHEN** an ordinary snapshot restore with a native version-2 lifetime commits its consuming Session/native owner pair +- **THEN** its immutable receipt SHALL bind the exact lifetime and complete retained checkout-set digest, rechecking the set at the paired writer before commit +- **AND** lost or changed resource proof SHALL refuse the complete consuming pair and native acknowledgement, while lookup SHALL verify the bound lifetime against original native history +- **AND** historical receipts without checkout evidence SHALL remain readable without acquiring checkout acknowledgement or reclamation authority + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index c1d7d0079..33a852dbb 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -345,3 +345,13 @@ engine. This lifts only the metadata reservation refusal; ordinary owned Git source availability remains false. Actual Git worker/checkout consuming receipts, post-effect acknowledgement/reconciliation, reclamation and complete crash/client/ platform acceptance remain required; M2–M8 and parent tasks remain open. + + +Ordinary consuming-receipt continuation: new restore receipts now bind exact +version-2 lifetime and complete checkout-set digest when native lifetime proof +exists. The paired writer rechecks the retained set and lookup verifies original +native lifetime history. Legacy receipt encoding remains unchanged without the +optional field and cannot generate new checkout acknowledgement evidence. This +adds receipt evidence only; exact post-effect native checkout publication and +reconciliation, actual Git runtime acceptance, reclamation and remaining M2–M8 / +parent obligations remain open. Git source availability stays disabled. diff --git a/sdk/openapi.json b/sdk/openapi.json index 6a1c30b0f..f1a79158e 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7528,6 +7528,17 @@ }, "events_digest": { "type": "string" + }, + "checkout": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRestoreCheckout" + }, + { + "type": "null" + } + ], + "description": "Missing on historical receipts; absence never authorizes checkout acknowledgement." } }, "required": [ @@ -7539,6 +7550,98 @@ ], "description": "Exact native intent and Session batch consumed by an ordinary restore.\nThis is durable acknowledgement evidence, never authority to replay files." }, + "SnapshotRestoreCheckout": { + "type": "object", + "additionalProperties": false, + "properties": { + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "lifetime": { + "$ref": "#/components/schemas/SnapshotLifetimeBinding" + }, + "resources_digest": { + "type": "string" + } + }, + "required": [ + "version", + "lifetime", + "resources_digest" + ] + }, + "SnapshotLifetimeBinding": { + "type": "object", + "additionalProperties": false, + "properties": { + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "owner": { + "$ref": "#/components/schemas/SnapshotLifetimeRequest" + }, + "root_identity": { + "type": "string" + }, + "namespace_identity": { + "type": "string" + }, + "directory_identity": { + "type": "string" + }, + "file_identity": { + "type": "string" + }, + "source_phase": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "version", + "owner", + "root_identity", + "namespace_identity", + "directory_identity", + "file_identity" + ] + }, + "SnapshotLifetimeRequest": { + "type": "object", + "additionalProperties": false, + "properties": { + "activity_id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "session_created_id": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "worktree_id": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "activity_id", + "session_id", + "session_created_id", + "directory" + ] + }, "Data_Array_of_SnapshotPreparation": { "type": "object", "properties": { @@ -7632,76 +7735,6 @@ "warning" ] }, - "SnapshotLifetimeBinding": { - "type": "object", - "additionalProperties": false, - "properties": { - "version": { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - "owner": { - "$ref": "#/components/schemas/SnapshotLifetimeRequest" - }, - "root_identity": { - "type": "string" - }, - "namespace_identity": { - "type": "string" - }, - "directory_identity": { - "type": "string" - }, - "file_identity": { - "type": "string" - }, - "source_phase": { - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "version", - "owner", - "root_identity", - "namespace_identity", - "directory_identity", - "file_identity" - ] - }, - "SnapshotLifetimeRequest": { - "type": "object", - "additionalProperties": false, - "properties": { - "activity_id": { - "type": "string" - }, - "session_id": { - "type": "string" - }, - "session_created_id": { - "type": "string" - }, - "directory": { - "type": "string" - }, - "worktree_id": { - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "activity_id", - "session_id", - "session_created_id", - "directory" - ] - }, "SnapshotPreparationConfirmation": { "type": "object", "additionalProperties": false, diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 5fae75ad4..f862df659 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -4876,6 +4876,12 @@ export type SnapshotRecoverySelection = { export type SnapshotRecoveryState = "pending" | "unknown" | "acknowledged" | "resolved_by_another_request" | "cancelled_before_origin" | "cancelled_before_effects"; +export type SnapshotRestoreCheckout = { + version: number; + lifetime: SnapshotLifetimeBinding; + resources_digest: string; +}; + /** * Exact native intent and Session batch consumed by an ordinary restore. * This is durable acknowledgement evidence, never authority to replay files. @@ -4886,6 +4892,8 @@ export type SnapshotRestoreReceipt = { session_before: number; session_after: number; events_digest: string; + /** Missing on historical receipts; absence never authorizes checkout acknowledgement. */ + checkout?: SnapshotRestoreCheckout | null; }; export type SolveCandidateEvidence = { From 08bc1d9ec63a7d159c4272a537f62d0c4161e221 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 04:34:50 -0400 Subject: [PATCH 29/87] Acknowledge exact restore checkout claims after durable consuming commits --- .../src/worktrees/activity/snapshot.rs | 13 +- .../activity/snapshot/acknowledgement.rs | 2 +- .../tests/worktree_repository/activity.rs | 54 ++++ crates/cyber-server/src/runtime/activity.rs | 29 +- crates/cyber-server/src/runtime/host.rs | 28 ++ .../src/runtime/snapshot_acknowledgement.rs | 107 ++++++- .../src/runtime/snapshot_work_tests.rs | 265 ++++++++++++++++++ crates/cyber-tools/src/worktrees.rs | 102 +++++-- crates/cyber-tools/tests/worktree_setup.rs | 61 +++- .../implementation.md | 66 +++++ .../managed-recovery.md | 20 ++ .../specs/snapshots-checkpoints/spec.md | 11 + .../complete-daily-coding-surfaces/tasks.md | 13 + 13 files changed, 727 insertions(+), 44 deletions(-) diff --git a/crates/cyber-core/src/worktrees/activity/snapshot.rs b/crates/cyber-core/src/worktrees/activity/snapshot.rs index c5f9973c4..57058a38b 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot.rs @@ -45,12 +45,23 @@ pub struct CheckoutOwner { pub lifetime_digest: String, } -/// Read-only reacquired evidence, deliberately separate from `CheckoutLease`. +/// Retained owned or reacquired evidence, deliberately separate from `CheckoutLease`. /// No generic settlement, creation or source mutation capability is exposed. pub struct SnapshotClaim { lease: super::CheckoutLease, acknowledgement: Option, } +impl super::CheckoutLease { + /// Retain an owned snapshot claim without rewriting its unsettled evidence. + /// No generic settlement capability survives this conversion. + pub fn retain_snapshot(self) -> io::Result { + self.verify_snapshot()?; + Ok(SnapshotClaim { + lease: self, + acknowledgement: None, + }) + } +} impl SnapshotClaim { pub fn binding(&self) -> &CheckoutBinding { self.lease diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs index 3490cd594..dd2bafcb2 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs @@ -135,7 +135,7 @@ impl CheckoutAcknowledgement { receipt_digest: receipt_digest.into(), }) } - /// Native metadata only. The caller must retain matching exited lifetime and + /// Native metadata only. The caller must retain matching owned or exited lifetime and /// complete-resource proof through the observed Store commit and this call. pub fn publish(&self, claim: &mut SnapshotClaim) -> io::Result<()> { self.publish_before(claim, || Ok(())) diff --git a/crates/cyber-core/tests/worktree_repository/activity.rs b/crates/cyber-core/tests/worktree_repository/activity.rs index 013a52634..476ef1c2f 100644 --- a/crates/cyber-core/tests/worktree_repository/activity.rs +++ b/crates/cyber-core/tests/worktree_repository/activity.rs @@ -947,3 +947,57 @@ fn claims_refuse_stale_ownership_and_invalid_session_names_before_creating_recor assert!(block_on(repository.claim(&fixture.execution, &managed, "ses_valid")).is_err()); assert!(!managed.common_dir.join("cyber-worktree-activity").exists()); } + +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[test] +fn retaining_owned_snapshot_claim_never_settles_unknown_evidence_without_a_marker() { + use cyber_core::worktrees::{CheckoutAcknowledgement, CheckoutOwner}; + for acknowledge in [false, true] { + let fixture = Fixture::new(); + let repository = fixture.repository(); + let managed = fixture + .create( + &repository, + &Name::parse("owned-retain").unwrap(), + &Settings::default(), + ) + .unwrap(); + let owner = CheckoutOwner { + activity_id: "op_owned".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: managed.path.clone(), + lifetime_digest: "a".repeat(64), + }; + let lease = block_on(repository.claim_snapshot_bound(&fixture.execution, &managed, &owner)) + .unwrap(); + let path = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id) + .join("op_owned.lock"); + let original = std::fs::read(&path).unwrap(); + assert_eq!( + serde_json::from_slice::(&original).unwrap()["settled"], + false + ); + let mut claim = lease.retain_snapshot().unwrap(); + claim.verify().unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), original); + assert!( + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, true)) + .is_err() + ); + if acknowledge { + CheckoutAcknowledgement::new(claim.binding(), &"a".repeat(64), &"b".repeat(64)) + .unwrap() + .publish(&mut claim) + .unwrap(); + } + drop(claim); + let result = + block_on(repository.remove(&fixture.execution, &CheckoutActivity, &managed, true)); + assert_eq!(result.is_ok(), acknowledge); + assert_eq!(std::fs::read(&path).unwrap(), original); + } +} diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 5534d3310..5e51c1e22 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -341,7 +341,7 @@ impl Scope { sequence: i64, events: Vec, intent_digest: Option<&str>, - location: Box, + mut location: Box, ) -> Result, RuntimeError> { let runtime = Runtime { inner: self.inner.upgrade().ok_or(RuntimeError::ShuttingDown)?, @@ -373,6 +373,17 @@ impl Scope { ) }) .transpose()?; + if settled.snapshot_restore_receipt.is_none() + && self + .snapshot_lifetime() + .is_some_and(|lifetime| lifetime.owner.worktree_id.is_some()) + { + return Err(RuntimeError::Conflict( + "Managed snapshot completion lacks a consuming journal receipt; inspect retained recovery evidence".into(), + )); + } + let receipt = settled.snapshot_restore_receipt.clone(); + let candidate = receipt.clone(); let checkout = settled .snapshot_restore_receipt .as_ref() @@ -384,7 +395,7 @@ impl Scope { (self.id.clone(), self.sequence()), ]; let stop = self.control.stop.clone(); - let (stored, location) = self + let (stored, mut location) = self .store .append_pair_checked(owners, move |tx| { location @@ -405,6 +416,13 @@ impl Scope { )); } } + if let Some(receipt) = &candidate + && receipt.checkout.is_some() + { + location + .prepare_restore_acknowledgement(receipt) + .map_err(cyber_store::StoreError::InvalidAppend)?; + } if stop.is_cancelled() { return Err(cyber_store::StoreError::InvalidAppend( "Snapshot consuming acknowledgement was cancelled".into(), @@ -424,6 +442,13 @@ impl Scope { state.apply(event).map_err(RuntimeError::Corrupt)?; } runtime.inner.publish(&stored[0]); + if let Some(receipt) = &receipt + && receipt.checkout.is_some() + { + location.acknowledge_restore(receipt).map_err(|error| RuntimeError::Conflict(format!( + "Snapshot runtime acknowledgement committed; checkout acknowledgement requires reconciliation: {error}" + )))?; + } Ok(location) } pub fn settle(&mut self) -> Result<(), RuntimeError> { diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index b27349a04..2df01d6a4 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -63,6 +63,34 @@ pub trait SnapshotLocationGuard: Send { ) -> Result<(), String> { self.prepare_preparation_acknowledgement(receipt) } + + /// Read-only candidate check under retained exact checkout locks. + fn prepare_restore_acknowledgement( + &mut self, + receipt: &super::SnapshotRestoreReceipt, + ) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + let checkout = receipt + .checkout + .as_ref() + .ok_or("Missing restore checkout proof")?; + let resources = self.bindings()?; + if checkout.lifetime.owner.worktree_id.is_some() || !resources.is_empty() { + return Err("Native restore checkout acknowledgement is unsupported".into()); + } + if checkout.lifetime.checkout_resources_digest(&resources)? != checkout.resources_digest { + return Err("Receipt differs from retained restore resources".into()); + } + Ok(()) + } + + /// Metadata only, issued after observing the exact consuming runtime pair. + fn acknowledge_restore( + &mut self, + receipt: &super::SnapshotRestoreReceipt, + ) -> Result<(), String> { + self.prepare_restore_acknowledgement(receipt) + } } struct UnmanagedSnapshotLocation { _owner: Box, diff --git a/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs index 73fa11c8f..01d12e842 100644 --- a/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs +++ b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs @@ -14,11 +14,40 @@ pub struct SnapshotAcknowledgementWorker { origin: SnapshotOrigin, control: Arc, // Actual resources are released before ordinary admission or Stop waiters. - _location: Box, + _location: Option>, + checkouts: Option, _lifecycle: OwnedRwLockReadGuard<()>, _admission: Admission, _completion: Completion, } +struct RestoreResources { + claims: Box, + // Last: retain the original worker's lock until every native claim is released. + lifetime: Box, +} +impl RestoreResources { + fn verify(&self, receipt: &super::SnapshotRestoreReceipt) -> Result<(), String> { + let checkout = receipt + .checkout + .as_ref() + .ok_or("Missing restore checkout evidence")?; + if self.lifetime.binding() != &checkout.lifetime { + return Err("Restore acknowledgement lifetime changed".into()); + } + self.lifetime.verify()?; + let resources = self + .lifetime + .checkout_bindings()? + .ok_or("Missing published restore resources")?; + self.claims.verify()?; + if self.claims.bindings()? != resources + || checkout.lifetime.checkout_resources_digest(&resources)? != checkout.resources_digest + { + return Err("Original restore checkout resource set differs".into()); + } + Ok(()) + } +} #[derive(Clone)] pub(super) enum Acknowledgement { Recovery(SnapshotRecoveryReceipt), @@ -87,6 +116,11 @@ impl SnapshotAcknowledgementWork<'_> { if self.0.control.stop.is_cancelled() { return Err("Snapshot acknowledgement was cancelled".into()); } + if let (Some(resources), Acknowledgement::Restore(receipt)) = + (&self.0.checkouts, &self.0.receipt) + { + resources.verify(receipt)?; + } let authority = self.0.authority.clone(); let receipt = self.0.receipt.clone(); let origin = self.0.origin.clone(); @@ -117,11 +151,23 @@ impl SnapshotAcknowledgementWork<'_> { impl SnapshotAcknowledgementWorker { pub async fn run( - self, + mut self, work: impl FnOnce(&SnapshotAcknowledgementWork<'_>) -> Result<(), String> + Send + 'static, ) -> Result<(), String> { let cancelled = self.control.stop.clone().drop_guard(); - let joined = tokio::task::spawn_blocking(move || work(&SnapshotAcknowledgementWork(&self))); + let joined = tokio::task::spawn_blocking(move || { + work(&SnapshotAcknowledgementWork(&self))?; + if self.checkouts.is_some() { + SnapshotAcknowledgementWork(&self).authorize()?; + } + if let (Some(resources), Acknowledgement::Restore(receipt)) = + (&mut self.checkouts, &self.receipt) + { + resources.verify(receipt)?; + resources.claims.acknowledge_restore(receipt)?; + } + Ok(()) + }); let result = joined.await.map_err(|_| { "Snapshot acknowledgement worker failed; lookup and reconcile its committed receipt" .to_owned() @@ -214,15 +260,49 @@ impl Runtime { id, control: control.clone(), }; - let lease = self - .inner - .claim_location(&state.info, false, control.stop.clone()) - .await?; - if lease.worktree_id != state.info.worktree_id { - return Err(RuntimeError::Conflict( - "Acknowledgement checkout changed".into(), - )); - } + let (location, checkouts) = if let Acknowledgement::Restore(restore) = &receipt + && let Some(checkout) = &restore.checkout + { + let lifetime = self + .inner + .options + .snapshots + .lock_worker(checkout.lifetime.clone()) + .await + .map_err(RuntimeError::Conflict)?; + let published = lifetime + .checkout_bindings() + .map_err(RuntimeError::Conflict)? + .ok_or_else(|| { + RuntimeError::Conflict("Original restore resources are unpublished".into()) + })?; + let claims = self + .inner + .tools + .reacquire_snapshot_location(&state.info, &checkout.lifetime, &published) + .map_err(RuntimeError::Conflict)?; + let mut resources = RestoreResources { claims, lifetime }; + resources.verify(restore).map_err(RuntimeError::Conflict)?; + resources + .claims + .prepare_restore_acknowledgement(restore) + .map_err(RuntimeError::Conflict)?; + (None, Some(resources)) + } else { + let lease = self + .inner + .claim_location(&state.info, false, control.stop.clone()) + .await?; + if lease.worktree_id != state.info.worktree_id { + return Err(RuntimeError::Conflict( + "Acknowledgement checkout changed".into(), + )); + } + ( + Some(lease.settle_retained().map_err(RuntimeError::Invalid)?), + None, + ) + }; Ok(SnapshotAcknowledgementWorker { runtime: self.clone(), authority, @@ -230,7 +310,8 @@ impl Runtime { receipt, origin, control, - _location: lease.settle_retained().map_err(RuntimeError::Invalid)?, + _location: location, + checkouts, _lifecycle: lifecycle, _admission: admission, _completion: completion, diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 276a94344..2ea96bf1b 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -702,6 +702,13 @@ impl Snapshots for Lifetimes { as Box)) }) } + fn acknowledge_recovery( + &self, + directory: String, + worker: super::super::SnapshotAcknowledgementWorker, + ) -> futures::future::BoxFuture<'_, Result<(), String>> { + self.inner.acknowledge_recovery(directory, worker) + } fn track( &self, directory: &str, @@ -3135,3 +3142,261 @@ async fn ordinary_restore_receipt_binds_native_lifetime_and_rechecks_resources_a runtime.shutdown().await; } } + +struct RestoreCheckoutAck { + stage: &'static str, + prepared: Arc, + acknowledged: Arc, +} +impl super::super::SnapshotLocationGuard for RestoreCheckoutAck { + fn verify(&self) -> Result<(), String> { + Ok(()) + } + fn prepare_restore_acknowledgement( + &mut self, + receipt: &SnapshotRestoreReceipt, + ) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + self.prepared.fetch_add(1, Ordering::SeqCst); + if self.stage == "prepare" { + Err("fixture refused candidate checkout receipt".into()) + } else { + Ok(()) + } + } + fn acknowledge_restore(&mut self, receipt: &SnapshotRestoreReceipt) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + self.acknowledged.fetch_add(1, Ordering::SeqCst); + if self.stage == "publish" { + Err("fixture failed checkout marker publication".into()) + } else { + Ok(()) + } + } +} + +#[tokio::test] +async fn ordinary_checkout_acknowledgement_follows_the_consuming_pair_and_keeps_committed_state_on_failure() + { + for stage in ["valid", "prepare", "publish"] { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let reconciled = Arc::new(AtomicUsize::new(0)); + let resources = Arc::new(std::sync::Mutex::new(None)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(reconciled.clone()), + permits: Arc::new(tokio::sync::Semaphore::new(1)), + foreign: false, + reject_resources: false, + resources: resources.clone(), + }), + ); + let mut owner = worker(&runtime, &session).await; + let id = owner.scope.id().to_owned(); + let prepared = Arc::new(AtomicUsize::new(0)); + let acknowledged = Arc::new(AtomicUsize::new(0)); + owner._location = Some(Box::new(RestoreCheckoutAck { + stage, + prepared: prepared.clone(), + acknowledged: acknowledged.clone(), + })); + let binding = binding(temp.path().canonicalize().unwrap()); + let journal_acks = Arc::new(AtomicUsize::new(0)); + let seen = journal_acks.clone(); + let completed = owner + .run(move |work| { + let origin = work.begin(binding).unwrap(); + work.effect(|| Ok(())).unwrap(); + Ok(SnapshotNativeResult::new( + vec![], + Box::new(Guard { + origin: Some(origin), + acknowledgements: seen, + }), + )) + }) + .await + .unwrap(); + assert_eq!(prepared.load(Ordering::SeqCst), 0); + assert_eq!(acknowledged.load(Ordering::SeqCst), 0); + let handle = runtime.inner.handle(&session).await.unwrap(); + let mut state = handle.state.lock().await; + let before = state.last_seq; + let result = completed.commit( + &mut state, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "durable checkout".into(), + usage: None, + cost: None, + model: None, + }, + )], + ); + assert_eq!(prepared.load(Ordering::SeqCst), 1); + if stage == "prepare" { + assert!(result.is_err()); + assert_eq!(state.last_seq, before); + assert_eq!(acknowledged.load(Ordering::SeqCst), 0); + assert_eq!(journal_acks.load(Ordering::SeqCst), 0); + drop(state); + assert!( + runtime + .snapshot_restore_lookup(&session, &id) + .await + .unwrap() + .is_none() + ); + assert_eq!(latest(&runtime, &session)["status"], "unknown"); + } else { + if stage == "publish" { + assert!( + result + .unwrap_err() + .to_string() + .contains("runtime acknowledgement committed") + ); + } else { + result.unwrap(); + } + assert_eq!(state.info.title, "durable checkout"); + assert!(state.last_seq > before); + assert_eq!(acknowledged.load(Ordering::SeqCst), 1); + assert_eq!( + journal_acks.load(Ordering::SeqCst), + usize::from(stage == "valid") + ); + drop(state); + assert!( + runtime + .snapshot_restore_lookup(&session, &id) + .await + .unwrap() + .unwrap() + .checkout + .is_some() + ); + assert_eq!(latest(&runtime, &session)["status"], "settled"); + if stage == "publish" { + let before = runtime + .snapshot_restore_lookup(&session, &id) + .await + .unwrap() + .unwrap(); + *resources.lock().unwrap() = None; + assert!( + runtime + .snapshot_restore_acknowledge(&session, &id) + .await + .is_err() + ); + assert_eq!(reconciled.load(Ordering::SeqCst), 0); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); + *resources.lock().unwrap() = Some(vec![]); + let reconciled_receipt = runtime + .snapshot_restore_acknowledge(&session, &id) + .await + .unwrap(); + assert_eq!(reconciled_receipt, before); + runtime + .snapshot_restore_acknowledge(&session, &id) + .await + .unwrap(); + assert_eq!(reconciled.load(Ordering::SeqCst), 2); + assert_eq!( + journal_acks.load(Ordering::SeqCst), + 0, + "metadata retry replayed the original worker" + ); + assert_eq!( + runtime.state(&session).await.unwrap().info.title, + "durable checkout" + ); + } + } + runtime.shutdown().await; + } +} + +#[tokio::test] +async fn managed_completion_without_a_journal_receipt_keeps_pre_effect_recovery_evidence() { + for managed in [false, true] { + let (temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let handle = runtime.inner.handle(&session).await.unwrap(); + let created = runtime + .inner + .store + .read_events(&session, -1, 1) + .unwrap() + .events[0] + .id + .clone(); + let id = cyber_core::ids::new_id("op"); + // Writer-boundary fixture: no native source worker or checkout is admitted. + let lifetime = super::super::SnapshotLifetimeBinding { + version: 2, + owner: super::super::SnapshotLifetimeRequest { + activity_id: id.clone(), + session_id: session.clone(), + session_created_id: created, + directory: temp.path().canonicalize().unwrap(), + worktree_id: managed.then(|| "wt_fixture".into()), + }, + root_identity: "fixture-root".into(), + namespace_identity: "fixture-namespace".into(), + directory_identity: "fixture-directory".into(), + file_identity: "fixture-file".into(), + source_phase: Some("phase_fixture".into()), + }; + let mut scope = activity::Scope::reserve_snapshot_owned( + &runtime.inner, + session.clone(), + id, + Some(lifetime), + ) + .await + .unwrap(); + let mut state = handle.state.lock().await; + let before = state.last_seq; + let result = scope.commit_snapshot( + &mut state, + before, + vec![events::event( + events::TITLE_GENERATED, + &events::Titled { + title: "no source effects".into(), + usage: None, + cost: None, + model: None, + }, + )], + None, + Box::new(BindingLease { + bindings: vec![], + settlements: Arc::new(AtomicUsize::new(0)), + }), + ); + if managed { + assert!(result.is_err()); + assert_eq!(state.last_seq, before); + } else { + result.unwrap(); + assert_eq!(state.info.title, "no source effects"); + } + drop(state); + drop(scope); + let record = latest(&runtime, &session); + assert_eq!( + record["status"], + if managed { "unknown" } else { "settled" } + ); + assert!(record["snapshot_origin"].is_null()); + assert_eq!(record["snapshot_source_admitted"], false); + assert!(record["snapshot_restore_receipt"].is_null()); + runtime.shutdown().await; + } +} diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index 346d7f81c..d0e396db3 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -312,7 +312,7 @@ impl BuiltinHost { .map_err(|error| error.to_string()) }) .collect::, _>>()?; - Ok(Box::new(ReacquiredSnapshotProof(claims))) + Ok(Box::new(RetainedSnapshotProof(claims))) } pub(crate) async fn claim_hook_test_location( @@ -1310,10 +1310,22 @@ impl cyber_server::runtime::SnapshotLocationGuard for ActiveSnapshotLocation { fn bindings(&self) -> Result, String> { self.proof.bindings() } + fn prepare_restore_acknowledgement( + &mut self, + receipt: &cyber_server::runtime::SnapshotRestoreReceipt, + ) -> Result<(), String> { + self.proof.prepare_restore_acknowledgement(receipt) + } + fn acknowledge_restore( + &mut self, + receipt: &cyber_server::runtime::SnapshotRestoreReceipt, + ) -> Result<(), String> { + self.proof.acknowledge_restore(receipt) + } } -struct ReacquiredSnapshotProof(Vec); -impl cyber_server::runtime::SnapshotLocationGuard for ReacquiredSnapshotProof { +struct RetainedSnapshotProof(Vec); +impl cyber_server::runtime::SnapshotLocationGuard for RetainedSnapshotProof { fn verify(&self) -> Result<(), String> { self.0 .iter() @@ -1365,27 +1377,57 @@ impl cyber_server::runtime::SnapshotLocationGuard for ReacquiredSnapshotProof { } self.verify() } -} - -struct SnapshotWorktreeProof(Vec); -impl cyber_server::runtime::SnapshotLocationGuard for SnapshotWorktreeProof { - fn verify(&self) -> Result<(), String> { - for lease in &self.0 { - lease.verify_snapshot().map_err(|error| error.to_string())?; + fn prepare_restore_acknowledgement( + &mut self, + receipt: &cyber_server::runtime::SnapshotRestoreReceipt, + ) -> Result<(), String> { + let checkout = receipt + .checkout + .as_ref() + .ok_or("Legacy restore receipt lacks checkout evidence")?; + let digest = checkout + .lifetime + .checkout_resources_digest(&self.bindings()?)?; + if digest != checkout.resources_digest { + return Err("Restore checkout resources differ".into()); } - Ok(()) + let receipt_digest = receipt.checkout_acknowledgement_digest()?; + for claim in &mut self.0 { + cyber_core::worktrees::CheckoutAcknowledgement::new( + claim.binding(), + &digest, + &receipt_digest, + ) + .and_then(|ack| ack.retain_existing(claim)) + .map_err(|error| error.to_string())?; + } + self.verify() } - fn bindings(&self) -> Result, String> { - self.0 - .iter() - .map(|lease| { - lease.verify_snapshot().map_err(|error| error.to_string())?; - lease - .snapshot_binding() - .cloned() - .map_err(|error| error.to_string()) - }) - .collect() + fn acknowledge_restore( + &mut self, + receipt: &cyber_server::runtime::SnapshotRestoreReceipt, + ) -> Result<(), String> { + let checkout = receipt + .checkout + .as_ref() + .ok_or("Legacy restore receipt lacks checkout evidence")?; + let digest = checkout + .lifetime + .checkout_resources_digest(&self.bindings()?)?; + if digest != checkout.resources_digest { + return Err("Restore checkout resources differ".into()); + } + let receipt_digest = receipt.checkout_acknowledgement_digest()?; + for claim in &mut self.0 { + cyber_core::worktrees::CheckoutAcknowledgement::new( + claim.binding(), + &digest, + &receipt_digest, + ) + .and_then(|ack| ack.publish(claim)) + .map_err(|error| error.to_string())?; + } + self.verify() } } @@ -1426,9 +1468,21 @@ impl cyber_server::runtime::LocationGuard for WorktreeLease { let leases = self .0 .into_iter() - .map(|lease| lease.settle_retained().map_err(|error| error.to_string())) + .map(|lease| { + let lease = if lease + .snapshot_binding() + .map_err(|error| error.to_string())? + .version + == 1 + { + lease.settle_retained().map_err(|error| error.to_string())? + } else { + lease + }; + lease.retain_snapshot().map_err(|error| error.to_string()) + }) .collect::, _>>()?; - let proof = SnapshotWorktreeProof(leases); + let proof = RetainedSnapshotProof(leases); cyber_server::runtime::SnapshotLocationGuard::verify(&proof)?; Ok(Box::new(proof)) } diff --git a/crates/cyber-tools/tests/worktree_setup.rs b/crates/cyber-tools/tests/worktree_setup.rs index 8d93ce478..b939a7b73 100644 --- a/crates/cyber-tools/tests/worktree_setup.rs +++ b/crates/cyber-tools/tests/worktree_setup.rs @@ -1981,9 +1981,60 @@ async fn lifetime_bound_checkout_host_preserves_context_and_refuses_foreign_requ bindings[0].lifetime, Some(lifetime.checkout_owner().unwrap()) ); - let proof = lease.settle_snapshot_retained().unwrap(); + let original = std::fs::read(directory.join("op_bound.lock")).unwrap(); + let mut proof = lease.settle_snapshot_retained().unwrap(); proof.verify().unwrap(); assert_eq!(proof.bindings().unwrap(), bindings); + assert_eq!( + std::fs::read(directory.join("op_bound.lock")).unwrap(), + original + ); + // Host-adapter fixture: the runtime suites observe the actual consuming pair. + let restore = cyber_server::runtime::SnapshotRestoreReceipt { + origin: cyber_server::runtime::SnapshotOrigin { + version: 1, + session_id: info.id.clone(), + session_created_id: lifetime.owner.session_created_id.clone(), + activity_id: lifetime.owner.activity_id.clone(), + worktree_id: Some(managed.id.clone()), + binding: cyber_server::runtime::SnapshotBinding { + directory: managed.path.clone(), + root_identity: lifetime.root_identity.clone(), + namespace_identity: lifetime.namespace_identity.clone(), + journal_id: "restore_fixture".into(), + journal_identity: "fixture-journal".into(), + lock_identity: Some("fixture-lock".into()), + }, + }, + intent_digest: "b".repeat(64), + session_before: 8, + session_after: 9, + events_digest: "c".repeat(64), + checkout: Some(Box::new(cyber_server::runtime::SnapshotRestoreCheckout { + version: 1, + lifetime: lifetime.clone(), + resources_digest: lifetime.checkout_resources_digest(&bindings).unwrap(), + })), + }; + let marker = directory.join("op_bound.snapshot-ack"); + proof.prepare_restore_acknowledgement(&restore).unwrap(); + assert!(!marker.exists()); + proof.acknowledge_restore(&restore).unwrap(); + let acknowledged = std::fs::read(&marker).unwrap(); + proof.acknowledge_restore(&restore).unwrap(); + assert_eq!(std::fs::read(&marker).unwrap(), acknowledged); + let native: serde_json::Value = serde_json::from_slice(&acknowledged).unwrap(); + assert_eq!( + native["receipt_digest"], + restore.checkout_acknowledgement_digest().unwrap() + ); + let mut different = restore.clone(); + different.events_digest = "d".repeat(64); + assert!(proof.prepare_restore_acknowledgement(&different).is_err()); + let mut legacy = restore; + legacy.checkout = None; + assert!(proof.acknowledge_restore(&legacy).is_err()); + drop(proof); let raw: serde_json::Value = serde_json::from_slice(&std::fs::read(directory.join("op_bound.lock")).unwrap()).unwrap(); @@ -1991,8 +2042,12 @@ async fn lifetime_bound_checkout_host_preserves_context_and_refuses_foreign_requ raw["snapshot_binding"], serde_json::to_value(&bindings[0]).unwrap() ); - assert_eq!(raw["settled"], true); - assert_eq!(std::fs::read_dir(&directory).unwrap().count(), before + 1); + assert_eq!(raw["settled"], false); + assert_eq!( + std::fs::read(directory.join("op_bound.lock")).unwrap(), + original + ); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), before + 2); let mut exited = lifetime.clone(); exited.owner.activity_id = "op_exited".into(); let lease = flow diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 8c057e914..5027c2178 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2198,3 +2198,69 @@ diff and SDK generation consistency passed. Strict pinned OpenSpec passed all publication, CLI/TUI/App, Linux/Windows, full workspace and source-bound CI acceptance were not run here. No post-effect native checkout marker is published by this increment and no parent recovery task is accepted. + + +## Ordinary native checkout publication and reconciliation (2026-11-07) + +A consuming ordinary restore now verifies its candidate receipt through the retained +Location guard inside the paired writer, before any acknowledgement metadata is +created. After the durable pair is observed and Session state applied, the original +claims publish idempotent markers bound to that receipt's complete-resource digest +and versioned checksum. Candidate failure refuses the pair; publication failure +reports committed runtime state and leaves exact original evidence for explicit +metadata reconciliation. Journal acknowledgement follows successful checkout +publication in the ordinary completion path. Existing foreign markers refuse. + +The actual host now consumes owned CheckoutLease values into descriptor-retaining +SnapshotClaim proofs. Version-2 acquisition no longer calls generic settlement: +original UseRecord bytes remain unsettled, so worker exit without a consuming marker +still blocks native removal. Version-1 acquisition retains its former settled +encoding without gaining new marker authority. Core conversion performs no writes +and exposes no generic settlement method. Live and reacquired claims share the same +native verification/publication engine; the live activity wrapper delegates the new +receipt callbacks while retaining its LSP ownership. + +Explicit reconciliation of a new receipt locks its original lifetime, reads its +published complete set, reacquires exact native claims and checks existing markers. +The acknowledgement worker verifies native resources and durable authority before +journal metadata work and again before checkout publication. No new Session claim +is substituted for the original resources. Original claims are released before the +original lifetime lock; admission/Stop completion follow resource release. Legacy +receipt and recovery-receipt metadata paths retain their existing behavior. + +A journal-less managed completion cannot yet produce a positive consuming checkout +receipt. Committing it after preserving unsettled claims would leave an unresolvable +settled owner, so this path refuses before the pair and retains pre-effect/unknown +evidence instead. This is an interim refusal, not accepted managed no-op behavior; +a positive no-effect protocol remains required before ordinary Git admission. +Recovery-resolution checkout receipts/publication are also still incomplete. + +Native core coverage adds an owned-claim/removal regression with and without a +matching marker, asserting original bytes stay identical. The actual host test now +checks unsettled acquisition bytes, read-only candidate verification, ordinary marker +publication/retry, foreign receipt and legacy refusal, plus existing pre-effect +acknowledgement coverage. Its lifetime/receipt inputs are adapter fixtures; it is not +a real consuming Store/runtime/source proof. Instrumented runtime tests exercise +candidate refusal, committed publication failure, successful ordering, repeated +explicit reconciliation without source replay, missing original publication refusal, +and journal-less managed refusal versus unchanged unmanaged no-effect behavior. +Actual Git backend source callbacks and managed no-op positives remain unverified. + +The code-review skill was applied to every source/test/artifact change, focusing on +preserved unknown/removal fences, read-only writer callbacks, original ownership +versus generic settlement, durable-commit publication ordering, partial/idempotent +markers, legacy limits, cancellation/resource disposal, and metadata-only original +resource reacquisition. Admission, cleanup and full parent acceptance remain gated. + +Final local macOS arm64 verification: locked core worktree-repository tests passed +74 cases with one ignored fixture, actual host worktree-setup tests passed all 40, +and server library tests passed 432 cases with one ignored fixture. Four-package +all-target locked Clippy (core/server/snapshot/tools) passed with Rust warnings +denied. Native C compilation emitted sandbox_init/sandbox_free_error deprecation +warnings; no main comparison or pre-existing attribution was made. Format/diff and +generated SDK consistency passed. Strict pinned OpenSpec passed all 117 items; +spec lint reported 0 errors/25 warnings. Full snapshot/SDK/workspace execution, +real CLI/TUI/App, actual Git backend consuming callbacks, Linux/Windows and +source-bound CI acceptance were not run for this increment. No-op success, +recovery-resolution checkout acknowledgements and reference-aware cleanup remain +open, as do all parent recovery acceptance tasks. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 70a2ad621..a0dd0ea32 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -578,3 +578,23 @@ lookup SHALL verify the receipt's lifetime against the original native history. Historical receipts without this field remain readable without gaining checkout acknowledgement authority. The receipt field is evidence for subsequent exact native acknowledgement, not publication or reclamation authority by itself. + +Version-2 checkout acquisition SHALL retain the owned original claim without +rewriting its unsettled record before source dispatch. An exited source worker +without a consuming acknowledgement SHALL therefore continue fencing native +removal. The consuming writer SHALL check a proposed ordinary-restore receipt +against retained claims and any existing disposition markers using read-only +verification. Only after observing the durable pair may the native host publish +idempotent markers for that receipt. Partial publication retains original bytes +and remaining fences; a failure SHALL report committed runtime state and require +explicit metadata reconciliation, never source replay. Legacy acquisition retains +its original behavior without gaining version-2 acknowledgement authority. + +Explicit reconciliation for a new ordinary restore receipt SHALL reacquire the +receipt's original lifetime and complete checkout set, revalidate durable owner +history and current authority, and publish only journal/checkout acknowledgement +metadata. Missing publication, live locks, changed resources or foreign markers +SHALL refuse. Legacy receipts retain their original metadata path without gaining +claim adoption. Managed completions lacking a journal receipt SHALL retain their +pre-effect/unknown evidence instead of committing an unacknowledgeable claim; a +positive no-effect consuming protocol remains required before ordinary admission. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 2df161a04..31c8306ce 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -271,6 +271,17 @@ - **AND** lost or changed resource proof SHALL refuse the complete consuming pair and native acknowledgement, while lookup SHALL verify the bound lifetime against original native history - **AND** historical receipts without checkout evidence SHALL remain readable without acquiring checkout acknowledgement or reclamation authority +#### Scenario: Owned checkout evidence survives source dispatch and consuming commit +- **WHEN** a version-2 snapshot worker retains an acquired managed checkout +- **THEN** acquisition SHALL preserve its unsettled original claim bytes and native lock, so an exited worker without acknowledgement continues to fence removal +- **AND** the consuming writer SHALL read-only verify the exact proposed restore receipt against retained claims before committing, then publish only matching idempotent markers after observing the durable pair +- **AND** failed publication SHALL preserve committed Session state and original claims for explicit reconciliation, with no source replay or unrelated-claim acknowledgement + +#### Scenario: Ordinary checkout acknowledgement reconciles original resources +- **WHEN** an owner explicitly reconciles a committed ordinary restore with bound checkout evidence +- **THEN** reconciliation SHALL reacquire the exact original lifetime and complete checkout set, refuse missing/changed/live proof, and perform only bounded journal and checkout acknowledgement metadata writes +- **AND** legacy receipts SHALL gain no claim adoption authority and a managed completion without a journal receipt SHALL NOT manufacture one or discard its recovery fence + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 33a852dbb..667f00802 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -355,3 +355,16 @@ optional field and cannot generate new checkout acknowledgement evidence. This adds receipt evidence only; exact post-effect native checkout publication and reconciliation, actual Git runtime acceptance, reclamation and remaining M2–M8 / parent obligations remain open. Git source availability stays disabled. + + +Ordinary checkout acknowledgement continuation: version-2 host acquisition now +retains original unsettled claim bytes instead of applying generic settlement +before source dispatch. The consuming writer verifies candidate markers read-only; +observed durable ordinary-restore receipts publish exact native markers. Explicit +reconciliation reacquires original lifetime/resources and never dispatches source +work. Legacy version-1 acquisition and old receipt behavior remain distinct. Missing +managed journal receipts refuse with retained recovery evidence; successful managed +no-effect completion is not implemented. Recovery-resolution receipt/checkout +integration, actual Git source/backend/runtime and complete crash/platform/client +acceptance, reference-aware cleanup and other M2–M8/parent obligations remain open. +Ordinary Git source admission remains disabled. From 9484b9c4fa6bd9bcf1d1f695a15176992c56596d Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 04:49:28 -0400 Subject: [PATCH 30/87] Journal owned no-op restores and bind distinct native namespaces correctly --- crates/cyber-cli/tests/branches_cli.rs | 81 ++++++++++++++++++- .../src/runtime/snapshot_restore_receipt.rs | 3 +- .../src/runtime/snapshot_work_tests.rs | 6 +- crates/cyber-snapshot/src/copy/git.rs | 2 +- crates/cyber-snapshot/src/copy/restore.rs | 2 +- .../implementation.md | 70 ++++++++++++++++ .../managed-recovery.md | 9 +++ .../specs/snapshots-checkpoints/spec.md | 10 +++ .../complete-daily-coding-surfaces/tasks.md | 12 +++ 9 files changed, 189 insertions(+), 6 deletions(-) diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index fb36e271a..81d89f4c2 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1289,6 +1289,59 @@ fn reviewed_non_git_code_rewind_uses_private_snapshots_and_keeps_owner_files() { assert_eq!(model.requests().len(), requests); } +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn reviewed_non_git_noop_rewind_keeps_a_real_unadmitted_empty_journal() { + let model = scripted_model::ScriptedModel::start(reply); + let env = Env::new(&model); + std::fs::remove_dir_all(env.root.join("repo/.git")).unwrap(); + let id = env.exec(None, "WRITE-A"); + let first = env.user_ids(&id)[0].clone(); + // The owner already restored the target bytes before preview. Conversation + // rewind still requires an exact owned completion, without any source effect. + std::fs::remove_file(env.root.join("repo/a.txt")).unwrap(); + std::fs::write(env.root.join("repo/owner.txt"), "later owner file").unwrap(); + let requests = model.requests().len(); + let preview = rewind_preview(&env, &id, &first, "both"); + json(&confirm_rewind(&env, &id, &first, "both", &preview)); + assert!(env.user_ids(&id).is_empty()); + assert_eq!(env.file("a.txt"), None); + assert_eq!(env.file("owner.txt").as_deref(), Some("later owner file")); + let catalog = json(&env.cyber(&["api", "GET", &format!("/sessions/{id}/rewind/recovery")])); + let journals = catalog["data"].as_array().unwrap(); + assert_eq!(journals.len(), 1); + assert_eq!(journals[0]["filesystem_complete"], true); + assert!(journals[0]["paths"].as_array().unwrap().is_empty()); + let journal = journals[0]["id"].as_str().unwrap(); + assert_owned_snapshot_receipt(&env, &id, journal); + let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let intent: Value = serde_json::from_slice( + &std::fs::read(namespace.join("journals").join(journal).join("intent.json")).unwrap(), + ) + .unwrap(); + assert!(intent["changes"].as_array().unwrap().is_empty()); + assert!(intent["ownership"]["files"].as_array().unwrap().is_empty()); + let activity = intent["origin"]["activity_id"].as_str().unwrap(); + let route = format!("/sessions/{id}/rewind/restores/{activity}"); + let receipt = json(&env.cyber(&["api", "GET", &route])); + let phase_name = receipt["data"]["checkout"]["lifetime"]["source_phase"] + .as_str() + .unwrap(); + let phase: Value = + serde_json::from_slice(&std::fs::read(namespace.join("workers").join(phase_name)).unwrap()) + .unwrap(); + assert_eq!(phase["admitted"], false); + assert!(phase["origin"].is_null()); + reconcile_ordinary_restore_metadata(&env, &id, journal); + assert_eq!(env.file("owner.txt").as_deref(), Some("later owner file")); + assert_eq!(model.requests().len(), requests); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn reconcile_ordinary_restore_metadata(env: &Env, session: &str, journal: &str) { let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) @@ -1387,6 +1440,7 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { assert_snapshot_lifetime(&namespaces[0], &activity, origin); assert_eq!(activity["status"], "settled"); assert_eq!(activity["phase"], "snapshot_restore_committed"); + assert_restore_receipt_binding(&activity, origin); let created: String = db .query_row( "SELECT id FROM event WHERE aggregate_id=?1 ORDER BY seq LIMIT 1", @@ -1397,6 +1451,21 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { assert_eq!(origin["session_created_id"], created); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_restore_receipt_binding(activity: &Value, origin: &Value) { + let receipt = &activity["snapshot_restore_receipt"]; + assert_eq!(receipt["origin"], *origin); + assert_eq!( + receipt["checkout"]["lifetime"], + activity["snapshot_lifetime"] + ); + assert_ne!( + receipt["checkout"]["lifetime"]["namespace_identity"], + origin["binding"]["namespace_identity"], + "lifetime namespace and journal collection are distinct native directories" + ); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_snapshot_lifetime(namespace: &std::path::Path, activity: &Value, origin: &Value) { let lifetime = &activity["snapshot_lifetime"]; @@ -1422,8 +1491,16 @@ fn assert_snapshot_lifetime(namespace: &std::path::Path, activity: &Value, origi ) .unwrap(); assert_eq!(phase["lifetime"], *lifetime); - assert_eq!(phase["admitted"], true); - assert_eq!(phase["origin"], *origin); + assert_eq!(phase["admitted"], activity["snapshot_source_admitted"]); + if phase["admitted"] == true { + assert_eq!(phase["origin"], *origin); + } else { + assert_eq!(phase["admitted"], false); + assert!( + phase["origin"].is_null(), + "no-op publication must not admit source effects" + ); + } // These real-service fixtures use unmanaged Locations. An explicitly // published empty set is distinct from missing constructor evidence. assert_eq!(phase["checkout_bindings"], serde_json::json!([])); diff --git a/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs index 5f6650f16..5aae3431b 100644 --- a/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs +++ b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs @@ -32,6 +32,8 @@ impl SnapshotRestoreReceipt { impl super::SnapshotRestoreCheckout { pub(super) fn validate(&self, origin: &SnapshotOrigin) -> Result<(), String> { + // The lifetime's Location namespace and the origin's journal collection + // are different directories. Native guards verify each route separately. let owner = &self.lifetime.owner; if self.version != 1 || self.lifetime.version != 2 @@ -43,7 +45,6 @@ impl super::SnapshotRestoreCheckout { || owner.worktree_id != origin.worktree_id || owner.directory != origin.binding.directory || self.lifetime.root_identity != origin.binding.root_identity - || self.lifetime.namespace_identity != origin.binding.namespace_identity || self.resources_digest.len() != 64 || !self .resources_digest diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 2ea96bf1b..7431b27d6 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -690,7 +690,7 @@ impl Snapshots for Lifetimes { version: 2, owner: request, root_identity: "instrumented_root".into(), - namespace_identity: "instrumented_namespace".into(), + namespace_identity: "fixture-lifetime-namespace".into(), directory_identity: "fixture-directory".into(), file_identity: "fixture-file".into(), source_phase: Some("phase_fixture".into()), @@ -3101,6 +3101,10 @@ async fn ordinary_restore_receipt_binds_native_lifetime_and_rechecks_resources_a .unwrap(); let checkout = receipt.checkout.as_ref().unwrap(); assert_eq!(checkout.lifetime, lifetime); + assert_ne!( + checkout.lifetime.namespace_identity, + receipt.origin.binding.namespace_identity + ); assert_eq!( checkout.resources_digest, lifetime.checkout_resources_digest(&[]).unwrap() diff --git a/crates/cyber-snapshot/src/copy/git.rs b/crates/cyber-snapshot/src/copy/git.rs index 207ab117f..3ca581609 100644 --- a/crates/cyber-snapshot/src/copy/git.rs +++ b/crates/cyber-snapshot/src/copy/git.rs @@ -120,7 +120,7 @@ impl Store { worker.authorize().map_err(RestoreError::Failed)?; } let (changes, context) = self.plan_git(&prepared)?; - if changes.is_empty() { + if changes.is_empty() && worker.is_none() { return Ok(vec![]); } let id = format!("restore_{}", ulid::Ulid::new()); diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index e70399794..9de79130d 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -388,7 +388,7 @@ impl Store { worker.authorize().map_err(RestoreError::Failed)?; } let (_, changes, context) = self.plan(target, recorded, expected, paths)?; - if changes.is_empty() { + if changes.is_empty() && worker.is_none() { return Ok(vec![]); } let id = format!("restore_{}", ulid::Ulid::new()); diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 5027c2178..e511c4403 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2264,3 +2264,73 @@ real CLI/TUI/App, actual Git backend consuming callbacks, Linux/Windows and source-bound CI acceptance were not run for this increment. No-op success, recovery-resolution checkout acknowledgements and reference-aware cleanup remain open, as do all parent recovery acceptance tasks. + + +## Real owned no-op journals and distinct native namespaces (2026-11-07) + +Both native restore adapters now publish a real empty journal when the revalidated +plan is empty and a worker is present. They use the normal bounded intent budget, +worker begin/origin, empty prepared ownership, durable completion and immutable +receipt path. The source-effect loop is empty, so neither database nor native source +admission occurs. No fake origin or journal identity is introduced. Ordinary +consuming receipts and checkout markers can therefore settle an owned no-op; the +writer still refuses a managed adapter that supplies no journal receipt. Legacy +unowned no-op calls still return without creating a journal. Git source admission +remains gated independently of this backend change. + +Real client verification exposed a receipt-validation mistake in the recent +checkout evidence work: lifetime.namespace_identity identifies the private Location +namespace, whereas origin.binding.namespace_identity identifies its journals +collection. Equating them caused actual owned copy restores to fail with 'Restore +receipt checkout ownership differs'. The existing real CLI code-rewind test failed +on unchanged local 08bc1d9e with that exact error before this fix. Earlier mock +fixtures used equal strings and missed the distinct native directories. This is +our local receipt regression, not a claim about main's result. + +Receipt validation now retains exact owner/Session creation/worktree/Location/root +checks while allowing the two independently verified namespaces to differ. Native +lifetime and journal guards continue validating their own paths, identities and +records; lookup still attests the original lifetime and receipt, and native checkout +resource digests still bind that lifetime. The runtime lifetime fixture deliberately +uses a different namespace and asserts the receipt retains it. + +A new real CLI/service/scripted-provider test puts source bytes at the rewind target +before preview, then commits conversation rewind through an empty owned journal. +It verifies retained origin/receipt, empty changes and ownership, false database and +native source admission, no native admitted origin, unchanged owner files and no +inference. It then resets acknowledgement metadata and twice reconciles the original +receipt after introducing a later owner symlink, verifying no source replay. Shared +real-client assertions now compare native source phase with durable activity +admission, allowing a real unadmitted empty intent without weakening normal effect +checks; they also assert distinct namespace identities in actual receipts. + +The code-review skill was applied to source/tests/artifacts, focusing on real versus +synthetic intent evidence, zero source admission, metadata budgeting, shared guard +completion and acknowledgement, separate namespace semantics, foreign proof +rejection and unchanged legacy/admission behavior. The real positive is file-copy +with an unmanaged Location; actual managed Git source/checkout no-op integration, +recovery-resolution resource acknowledgement, retention and complete client/ +platform/cut acceptance remain open. No parent recovery task is accepted here. + +Final local macOS arm64 verification: all 17 real CLI branch/recovery tests passed, +including actual partial-restore kill/restart keep/rollback, pre-origin/pre-source +cuts, restage/clear, disabled snapshots and the new no-op case. These remain local +unmanaged/file-copy and legacy-Git evidence, not owned managed-Git acceptance. +Locked snapshot tests passed 137 cases with two ignored fixtures (the pathspec +fixture is invoked by its parent); server library tests passed 432 cases with one +ignored fixture. The focused distinct-namespace receipt test also passed after +its explicit assertion was added. Three-package all-target locked Clippy +(server/snapshot/CLI) passed with Rust warnings denied. Native C build emitted two +sandbox API deprecation warnings; no main comparison or pre-existing attribution +was made. Clippy initially reported cognitive complexity 17 against limit 15 for +the expanded receipt assertion helper. The cognitive-complexity skill's extraction +workflow was applied using Rust's native Clippy analyzer (Rust is outside that +skill's dispatcher); splitting receipt binding assertions removed the violation. +The four affected real CLI cases passed again after this helper refactor. + +Final format/diff and generated SDK consistency passed. Pinned strict OpenSpec +passed all 117 items; spec lint reported 0 errors/25 warnings. Full workspace/SDK +execution, real TUI/App, owned managed-Git source/checkout callbacks, Linux/Windows +and source-bound CI were not run for this increment. Native recovery-resolution +resource acknowledgement and reference-aware cleanup remain required; parent +tasks and ordinary Git admission stay open/gated. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index a0dd0ea32..70026fc9c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -598,3 +598,12 @@ SHALL refuse. Legacy receipts retain their original metadata path without gainin claim adoption. Managed completions lacking a journal receipt SHALL retain their pre-effect/unknown evidence instead of committing an unacknowledgeable claim; a positive no-effect consuming protocol remains required before ordinary admission. + +An owned no-op restore SHALL publish a real bounded empty intent after complete +planning and revalidation. Its native origin, empty ownership section and durable +completion SHALL use the same consuming-receipt/checkout acknowledgement path as +other owned restores, without calling source admission or installing files. An +empty plan does not justify a synthetic journal ID or invented historical proof. +Legacy unowned no-op calls retain their former no-journal behavior. Lifetime and +journal namespace identities refer to different native directories; each is +verified by its native guard, and the receipt binds both rather than equating them. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 31c8306ce..66ab31d40 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -282,6 +282,16 @@ - **THEN** reconciliation SHALL reacquire the exact original lifetime and complete checkout set, refuse missing/changed/live proof, and perform only bounded journal and checkout acknowledgement metadata writes - **AND** legacy receipts SHALL gain no claim adoption authority and a managed completion without a journal receipt SHALL NOT manufacture one or discard its recovery fence +#### Scenario: Owned restore has no source changes after revalidation +- **WHEN** an owned restore validates the complete expected state and its plan has no source changes +- **THEN** it SHALL publish a real bounded empty intent with native origin, empty ownership evidence and durable completion, consuming its normal receipt and checkout acknowledgement without source admission or installation +- **AND** failed or lost consuming replies SHALL retain the same journal evidence for lookup/reconciliation, while legacy unowned no-op calls SHALL preserve their former no-journal behavior + +#### Scenario: Receipt binds distinct lifetime and journal namespaces +- **WHEN** a native restore's lifetime and journal guards verify their respective namespace routes +- **THEN** consuming receipt validation SHALL bind both exact identities and the common owner/Location/root context without requiring the distinct namespace directories to have equal identities +- **AND** lookup and acknowledgement SHALL retain the original lifetime and journal verification and reject foreign owner or changed resource proof + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 667f00802..5d009087f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -368,3 +368,15 @@ no-effect completion is not implemented. Recovery-resolution receipt/checkout integration, actual Git source/backend/runtime and complete crash/platform/client acceptance, reference-aware cleanup and other M2–M8/parent obligations remain open. Ordinary Git source admission remains disabled. + + +Owned no-op and native namespace continuation: both journal backends now retain a +real empty intent for an owned empty plan, using ordinary origin/completion and +receipt acknowledgement without source admission. Unowned no-op behavior stays +unchanged. Receipt validation no longer equates the lifetime Location namespace +with its distinct journal collection. Real CLI regressions cover owned file-copy +no-op completion, retained owner files, receipt lookup and repeated acknowledgement +without source replay/inference. The full managed Git backend/source/checkout path, +recovery-resolution resource acknowledgement, complete native cuts/platforms and +reference-aware cleanup still require acceptance; parent tasks and Git admission +remain unchanged and open. From b0cc52664fd8d70aaf607f53161bfd7841b68869 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 05:06:17 -0400 Subject: [PATCH 31/87] Retain original native recovery resources through resolution work --- .../cyber-server/src/runtime/snapshot_work.rs | 90 +++++++++++- .../src/runtime/snapshot_work_tests.rs | 128 +++++++++++++++++- .../implementation.md | 52 +++++++ .../managed-recovery.md | 9 ++ .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 11 ++ 6 files changed, 294 insertions(+), 2 deletions(-) diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 60d4d1587..9a955b8b8 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -108,6 +108,7 @@ pub struct SnapshotWorker { created_id: String, worktree_id: Option, _location: Option>, + _original: Option, _lifecycle: OwnedRwLockReadGuard<()>, // Last: releasing this lock proves all earlier worker resources were dropped. _lifetime: Option>, @@ -127,8 +128,38 @@ pub struct SnapshotWork<'a> { } pub struct SnapshotCompletion { - worker: SnapshotWorker, + // Native guard cleanup remains covered by the worker's lifetime locks. native: SnapshotNativeResult, + worker: SnapshotWorker, +} + +struct OriginalResources { + origin: SnapshotOrigin, + claims: Box, + bindings: Vec, + // Last: keep the exited original's lock through all native claim disposal. + lifetime: Box, +} +impl OriginalResources { + fn verify(&self) -> Result<(), String> { + self.lifetime.verify()?; + if self.lifetime.checkout_bindings()?.as_ref() != Some(&self.bindings) { + return Err("Original recovery checkout publication changed".into()); + } + self.claims.verify()?; + if self.claims.bindings()? != self.bindings { + return Err("Original recovery checkout ownership changed".into()); + } + super::SnapshotRestoreCheckout { + version: 1, + lifetime: self.lifetime.binding().clone(), + resources_digest: self + .lifetime + .binding() + .checkout_resources_digest(&self.bindings)?, + } + .validate(&self.origin) + } } impl SnapshotWork<'_> { @@ -175,6 +206,9 @@ impl SnapshotWork<'_> { if let Some(lifetime) = &self.owner._lifetime { lifetime.verify()?; } + if let Some(original) = &self.owner._original { + original.verify()?; + } if self.owner.scope.control.stop.is_cancelled() { return Err("Snapshot worker cancelled; retain its journal before retrying".into()); } @@ -362,6 +396,9 @@ impl SnapshotCompletion { .guard .verify(self.worker.scope.snapshot_origin()) .map_err(RuntimeError::Conflict)?; + if let Some(original) = &self.worker._original { + original.verify().map_err(RuntimeError::Conflict)?; + } let location = self .worker ._location @@ -476,6 +513,55 @@ impl Runtime { &request.original.session_created_id, &request.confirmation.request_id, ); + let original_origin = request.original.clone(); + let original_lifetime = self.inner.store.read(move |db| { + let (_, original) = super::snapshot_recovery_record::attest(db, &original_origin)?; + match original.snapshot_lifetime { + Some(binding) if binding.version == 2 => Ok(Some(binding)), + None => Ok(None), + Some(binding) if binding.version == 1 => Ok(None), + Some(_) => Err(cyber_store::StoreError::InvalidAppend( + "Unsupported original recovery lifetime version".into(), + )), + } + })?; + let original = if let Some(binding) = original_lifetime { + let lifetime = self + .inner + .options + .snapshots + .lock_worker(binding.clone()) + .await + .map_err(RuntimeError::Conflict)?; + if lifetime.binding() != &binding { + return Err(RuntimeError::Conflict( + "Original recovery lifetime differs".into(), + )); + } + let bindings = lifetime + .checkout_bindings() + .map_err(RuntimeError::Conflict)? + .ok_or_else(|| { + RuntimeError::Conflict( + "Original recovery resource publication is missing".into(), + ) + })?; + let claims = self + .inner + .tools + .reacquire_snapshot_location(&state.info, &binding, &bindings) + .map_err(RuntimeError::Conflict)?; + let original = OriginalResources { + origin: request.original.clone(), + claims, + bindings, + lifetime, + }; + original.verify().map_err(RuntimeError::Conflict)?; + Some(original) + } else { + None + }; let lifetime = self.snapshot_lifetime(state, &id).await?; let scope = activity::Scope::reserve_recovery( self, @@ -531,6 +617,7 @@ impl Runtime { created_id, worktree_id: state.info.worktree_id.clone(), _location: Some(location), + _original: original, _lifecycle: lifecycle, _lifetime: lifetime, }) @@ -601,6 +688,7 @@ impl Runtime { created_id, worktree_id, _location: Some(location), + _original: None, _lifecycle: lifecycle, _lifetime: lifetime, admission, diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 7431b27d6..0ff297863 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -1216,7 +1216,7 @@ async fn cancelling_an_unstarted_resolution_keeps_its_original_failed_restore_fe temp.path(), Arc::new(Lifetimes { inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), - permits: Arc::new(tokio::sync::Semaphore::new(1)), + permits: Arc::new(tokio::sync::Semaphore::new(2)), foreign: false, reject_resources: false, resources: Default::default(), @@ -3404,3 +3404,129 @@ async fn managed_completion_without_a_journal_receipt_keeps_pre_effect_recovery_ runtime.shutdown().await; } } + +#[tokio::test] +async fn resolution_retains_original_and_distinct_resolution_lifetimes_through_native_work() { + for changed in [false, true] { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let permits = Arc::new(tokio::sync::Semaphore::new(2)); + let resources = Arc::new(std::sync::Mutex::new(None)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: permits.clone(), + foreign: false, + reject_resources: false, + resources: resources.clone(), + }), + ); + let original = + failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + assert_eq!(permits.available_permits(), 2); + let request = recovery_request(original.clone()); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + assert_eq!(owner._original.as_ref().unwrap().origin, original); + assert_ne!(owner.scope.id(), original.activity_id); + assert_eq!( + permits.available_permits(), + 0, + "both owners must remain held" + ); + drop(state); + if changed { + *resources.lock().unwrap() = None; + let effects = Arc::new(AtomicUsize::new(0)); + let seen = effects.clone(); + let result = owner + .run(move |work| { + work.authorize().map_err(RestoreError::Failed)?; + seen.fetch_add(1, Ordering::SeqCst); + Err(RestoreError::Failed("unexpected dispatch".into())) + }) + .await; + assert!(result.is_err()); + assert_eq!(effects.load(Ordering::SeqCst), 0); + } else { + let completed = finish_recovery(owner, request, Arc::new(AtomicUsize::new(0))).await; + assert_eq!(permits.available_permits(), 0); + // No consuming pair: retaining owners alone cannot acknowledge either. + drop(completed); + } + assert_eq!(permits.available_permits(), 2); + let events = runtime + .inner + .store + .read_events(&original.activity_id, -1, 10) + .unwrap(); + assert_eq!(events.events.last().unwrap().data["status"], "unknown"); + runtime.shutdown().await; + } +} + +struct NativeDisposal { + permits: Arc, + disposed: Arc, +} +impl SnapshotNativeGuard for NativeDisposal { + fn verify(&self, origin: Option<&SnapshotOrigin>) -> Result<(), String> { + if origin.is_none() { + Ok(()) + } else { + Err("unexpected origin".into()) + } + } + fn acknowledge(&mut self, _: Option<&SnapshotOrigin>) -> Result<(), String> { + Ok(()) + } +} +impl Drop for NativeDisposal { + fn drop(&mut self) { + assert_eq!( + self.permits.available_permits(), + 0, + "native cleanup outlived lifetime ownership" + ); + self.disposed.store(true, Ordering::SeqCst); + } +} + +#[tokio::test] +async fn native_result_cleanup_precedes_worker_lifetime_exit() { + let (temp, base, session, _, _) = super::super::binary_documents_tests::fixture().await; + let permits = Arc::new(tokio::sync::Semaphore::new(1)); + let runtime = runtime_with_snapshots( + &base, + base.inner.store.clone(), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(Arc::new(AtomicUsize::new(0))), + permits: permits.clone(), + foreign: false, + reject_resources: false, + resources: Default::default(), + }), + ); + let owner = worker(&runtime, &session).await; + let disposed = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let pin = NativeDisposal { + permits: permits.clone(), + disposed: disposed.clone(), + }; + let completed = owner + .run(move |_| Ok(SnapshotNativeResult::new(vec![], Box::new(pin)))) + .await + .unwrap(); + assert_eq!(permits.available_permits(), 0); + drop(completed); + assert!(disposed.load(Ordering::SeqCst)); + assert_eq!(permits.available_permits(), 1); + runtime.shutdown().await; +} diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index e511c4403..41f128b37 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2334,3 +2334,55 @@ execution, real TUI/App, owned managed-Git source/checkout callbacks, Linux/Wind and source-bound CI were not run for this increment. Native recovery-resolution resource acknowledgement and reference-aware cleanup remain required; parent tasks and ordinary Git admission stay open/gated. + + +## Retaining interrupted-owner resources through resolution (2026-11-07) + +Recovery now reads the attested original activity's lifetime before creating the +resolution lifetime or publishing its owner. For version 2 it locks the exact +original lifetime, requires its complete published resource set, reacquires those +native claims read-only and verifies owner/Session creation/Location/root context. +The original lifetime and claims remain in the synchronous worker alongside the +distinct resolution owner's resources. Source authorization and recovery completion +revalidate original publication and retained bindings. No original claim is created, +settled or rewritten by acquisition. Version-1/missing historical lifetime evidence +stays distinct without new adoption; unsupported lifetime versions refuse. + +SnapshotCompletion now disposes its native result guard before its worker. This +keeps native guard cleanup under the existing lifetime locks rather than releasing +the worker proof first. Original claims drop before their original lifetime; the +resolution lifetime remains the worker's final ownership resource. Retention alone +commits no receipt and acknowledges no native fence. + +Two runtime regressions use instrumented lifetime/Location ports. One holds both +owners, checks distinct activity identity and verifies that changed original +publication prevents callback dispatch; dropping an unconsumed completion leaves +the original activity unknown. Another native result Drop guard asserts the worker +lifetime is still retained during its cleanup. The unstarted-resolution fixture +now provides two semaphore permits for the two independent owners; native storage +uses distinct per-activity lock files. These tests do not claim actual managed +checkout resource-set or native cross-process proof by themselves. + +The code-review skill was applied to source/tests/artifacts, focusing on acquisition +before dispatch, no retrospective legacy adoption, unsupported-version refusal, +read-only resource proof, lifetime/claim/native guard disposal order, cancellation +and preservation of unknown evidence. Full recovery receipts still need both +resource sets bound and consumed at the writer, with exact native marker publication +and lost-reply reconciliation. Safe cleanup, actual managed-Git callbacks and full +client/platform/cut acceptance remain open; no parent task or Git admission gate +is accepted by this retention increment. + +Final local macOS arm64 verification: locked server library tests passed 434 cases +with one ignored fixture; the complete 17-case real CLI branch/recovery suite +passed, including actual kill/restart, keep/rollback, stage/clear and no-op paths. +These clients use unmanaged file-copy and legacy Git paths, not owned managed-Git +source callbacks. The final focused worker suite passed 45 cases after the explicit +unsupported-version branch was added. Four-package all-target locked Clippy +(server/snapshot/tools/CLI) passed with Rust warnings denied. Native C compilation +reported sandbox_init/sandbox_free_error deprecation warnings; no main comparison +or pre-existing attribution was made. Format/diff and generated SDK consistency +passed. Pinned strict OpenSpec passed all 117 items; spec lint reported 0 errors/ +25 warnings. Full snapshot/SDK/workspace execution, actual managed checkout +cross-process retention, real TUI/App, Linux/Windows and source-bound CI were not +run for this increment. Consuming recovery resource evidence, exact native marker +publication/reconciliation and reference-aware cleanup remain open. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 70026fc9c..d9cf16e24 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -607,3 +607,12 @@ empty plan does not justify a synthetic journal ID or invented historical proof. Legacy unowned no-op calls retain their former no-journal behavior. Lifetime and journal namespace identities refer to different native directories; each is verified by its native guard, and the receipt binds both rather than equating them. + +Before dispatching a resolution, a version-2 interrupted owner SHALL have its +exact original lifetime and complete published checkout set reacquired read-only +and retained alongside the distinct resolution owner's resources. Verification +SHALL continue through source operations and consuming commit. Live, missing, +foreign or changed proof refuses before resolution dispatch. Legacy owners SHALL +NOT acquire retrospective checkout proof. Completion disposal SHALL release native +result guards before the worker's lifetime locks so guard cleanup cannot outlive +the native ownership proof. Retention alone grants no checkout acknowledgement. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 66ab31d40..85266291a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -292,6 +292,12 @@ - **THEN** consuming receipt validation SHALL bind both exact identities and the common owner/Location/root context without requiring the distinct namespace directories to have equal identities - **AND** lookup and acknowledgement SHALL retain the original lifetime and journal verification and reject foreign owner or changed resource proof +#### Scenario: Resolution retains the interrupted owner's native resources +- **WHEN** recovery prepares a resolution for an interrupted owner with version-2 lifetime proof +- **THEN** it SHALL reacquire and retain that original lifetime and complete published checkout set alongside the distinct resolution owner, refusing live, missing, foreign or changed proof before source dispatch +- **AND** source authorization and consuming completion SHALL revalidate original resources; legacy owners SHALL gain no retrospective proof and unknown lifetime versions SHALL refuse +- **AND** retaining owners alone SHALL NOT acknowledge either native activity, and native result guard disposal SHALL precede worker lifetime release + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 5d009087f..0b14d7a89 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -380,3 +380,14 @@ without source replay/inference. The full managed Git backend/source/checkout pa recovery-resolution resource acknowledgement, complete native cuts/platforms and reference-aware cleanup still require acceptance; parent tasks and Git admission remain unchanged and open. + + +Original-owner retention continuation: recovery now reads attested original lifetime +metadata, reacquires exact version-2 lifetime and published native checkout resources +before reserving a resolution owner, and retains both owners through work/commit. +Authorization revalidates the original set. Legacy v1/missing lifetimes gain no new +claim proof; unknown versions refuse. Completion drops native result guards before +worker lifetime locks. This is retention, not original/resolution checkout marker +authority: binding both sets into consuming recovery receipts and publishing/ +reconciling their exact acknowledgements remain required. M2–M8 and parent tasks +stay open; ordinary Git source admission is unchanged and disabled. From 0a7cfad453b29217f71f87db2372b7207fcfaa36 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 05:24:32 -0400 Subject: [PATCH 32/87] Bind and reconcile both recovery checkout owners --- crates/cyber-cli/tests/branches_cli.rs | 43 +++ crates/cyber-server/src/runtime/activity.rs | 44 +++- crates/cyber-server/src/runtime/host.rs | 25 ++ crates/cyber-server/src/runtime/mod.rs | 9 +- .../src/runtime/snapshot_acknowledgement.rs | 167 +++++++++++- .../src/runtime/snapshot_recovery.rs | 9 + .../src/runtime/snapshot_recovery_checkout.rs | 126 +++++++++ .../src/runtime/snapshot_recovery_types.rs | 19 ++ .../cyber-server/src/runtime/snapshot_work.rs | 95 ++++++- .../src/runtime/snapshot_work_tests.rs | 249 ++++++++++++++++++ crates/cyber-snapshot/src/copy/resolution.rs | 7 +- crates/cyber-tools/src/worktrees.rs | 63 +++++ crates/cyber-tools/tests/worktree_setup.rs | 178 +++++++++++++ docs/goal-delivery.md | 10 + .../implementation.md | 88 +++++++ .../managed-recovery.md | 10 + .../specs/snapshots-checkpoints/spec.md | 11 + .../complete-daily-coding-surfaces/tasks.md | 14 + sdk/openapi.json | 114 +++++--- sdk/typescript/src/generated/types.ts | 10 + 20 files changed, 1233 insertions(+), 58 deletions(-) create mode 100644 crates/cyber-server/src/runtime/snapshot_recovery_checkout.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 81d89f4c2..646e1bfd9 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1107,6 +1107,49 @@ fn assert_native_settlement_link(env: &Env, journal: &str, result: &Value) { assert_eq!(old["completed"], false); assert_eq!(old["acknowledged"], true); assert_eq!(old["settled_by"], result["receipt"]["resolution_id"]); + assert_recovery_checkout_lifetimes(&namespace, &result["receipt"]); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_recovery_checkout_lifetimes(namespace: &std::path::Path, receipt: &Value) { + let checkout = &receipt["checkout"]; + assert_eq!(checkout["version"], 1); + assert_ne!( + checkout["original"]["lifetime"]["owner"]["activity_id"], + checkout["resolution"]["lifetime"]["owner"]["activity_id"] + ); + for (role, origin) in [ + ("original", &receipt["original"]), + ("resolution", &checkout["resolution_origin"]), + ] { + let evidence = &checkout[role]; + let lifetime: cyber_server::runtime::SnapshotLifetimeBinding = + serde_json::from_value(evidence["lifetime"].clone()).unwrap(); + assert_eq!( + lifetime.owner.activity_id, + origin["activity_id"].as_str().unwrap() + ); + let recorded: Value = serde_json::from_slice( + &std::fs::read(namespace.join("workers").join(&lifetime.owner.activity_id)).unwrap(), + ) + .unwrap(); + assert_eq!(recorded, evidence["lifetime"]); + let phase: Value = serde_json::from_slice( + &std::fs::read( + namespace + .join("workers") + .join(lifetime.source_phase.as_ref().unwrap()), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(phase["lifetime"], evidence["lifetime"]); + assert_eq!(phase["checkout_bindings"], serde_json::json!([])); + assert_eq!( + evidence["resources_digest"], + lifetime.checkout_resources_digest(&[]).unwrap() + ); + } } #[cfg(any(target_os = "macos", target_os = "linux"))] diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index 5e51c1e22..bb87b3d60 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -471,8 +471,8 @@ impl Scope { state: &super::SessionState, sequence: i64, receipt: &super::SnapshotRecoveryReceipt, - location: Box, - ) -> Result, RuntimeError> { + mut location: super::snapshot_recovery_checkout::RecoveryClaims, + ) -> Result { self.verify_authority()?; let Authority::Recovery(authority) = &self.authority else { return Err(RuntimeError::Conflict( @@ -521,6 +521,7 @@ impl Scope { settled.snapshot_receipt = Some(receipt.clone()); let resolution_event = change(&settled); let authority = authority.clone(); + let published_receipt = receipt.clone(); let receipt = receipt.clone(); let expected_request = request.clone(); let stop = self.control.stop.clone(); @@ -528,15 +529,47 @@ impl Scope { (original.activity_id.clone(), original_sequence), (self.id.clone(), self.sequence()), ]; - let (_, location) = self + let (_, mut location) = self .store .append_pair_checked(owners, move |tx| { location - .verify() + .prepare(&receipt) .map_err(cyber_store::StoreError::InvalidAppend)?; let (_, mut original_record) = super::snapshot_recovery_record::attest(tx, &original)?; let (_, resolution_record) = super::snapshot_recovery_record::attest(tx, &origin)?; + if let Some(checkout) = &receipt.checkout { + receipt + .validate_checkout(&origin) + .map_err(cyber_store::StoreError::InvalidAppend)?; + if resolution_record.snapshot_lifetime.as_ref() + != Some(&checkout.resolution.lifetime) + || original_record + .snapshot_lifetime + .as_ref() + .filter(|life| life.version == 2) + != checkout + .original + .as_ref() + .map(|original| &original.lifetime) + { + return Err(cyber_store::StoreError::InvalidAppend( + "Recovery receipt lifetime history differs".into(), + )); + } + } else if original_record + .snapshot_lifetime + .as_ref() + .is_some_and(|life| life.version == 2) + || resolution_record + .snapshot_lifetime + .as_ref() + .is_some_and(|life| life.version == 2) + { + return Err(cyber_store::StoreError::InvalidAppend( + "Recovery receipt is missing lifetime evidence".into(), + )); + } let current: i64 = tx.query_row( "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", [&original.session_id], @@ -573,6 +606,9 @@ impl Scope { .acknowledge_snapshot_uncertainty(&request.original.activity_id); self.record = settled; self.settled = true; + location.acknowledge(&published_receipt).map_err(|error| RuntimeError::Conflict(format!( + "Recovery runtime acknowledgement committed; checkout acknowledgement requires reconciliation: {error}" + )))?; Ok(location) } pub async fn run(&self, future: impl std::future::Future) -> T { diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 2df01d6a4..48f2cf012 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -91,6 +91,31 @@ pub trait SnapshotLocationGuard: Send { ) -> Result<(), String> { self.prepare_restore_acknowledgement(receipt) } + fn prepare_recovery_acknowledgement( + &mut self, + receipt: &super::SnapshotRecoveryReceipt, + role: super::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + receipt.checkout_acknowledgement_digest()?; + let evidence = receipt + .checkout_evidence(role) + .ok_or("Missing recovery resource role")?; + let resources = self.bindings()?; + if evidence.lifetime.owner.worktree_id.is_some() || !resources.is_empty() { + return Err("Native recovery checkout acknowledgement is unsupported".into()); + } + if evidence.lifetime.checkout_resources_digest(&resources)? != evidence.resources_digest { + return Err("Recovery resource role differs from retained claims".into()); + } + Ok(()) + } + fn acknowledge_recovery( + &mut self, + receipt: &super::SnapshotRecoveryReceipt, + role: super::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + self.prepare_recovery_acknowledgement(receipt, role) + } } struct UnmanagedSnapshotLocation { _owner: Box, diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 21bb563e3..1aec4d2af 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -134,6 +134,7 @@ pub use snapshot_preparation::{ SnapshotPreparationReview, }; mod snapshot_recovery; +mod snapshot_recovery_checkout; mod snapshot_recovery_types; mod snapshot_restore_receipt; pub use snapshot_lifetime::{SnapshotLifetime, SnapshotLifetimeBinding, SnapshotLifetimeRequest}; @@ -143,10 +144,10 @@ pub use rewind_preview::{ RewindSelection, }; pub use snapshot_recovery_types::{ - SnapshotRecoveryAction, SnapshotRecoveryConfirmation, SnapshotRecoveryImage, - SnapshotRecoveryImageKind, SnapshotRecoveryPath, SnapshotRecoveryReceipt, - SnapshotRecoveryResult, SnapshotRecoveryReview, SnapshotRecoverySelection, - SnapshotRecoveryState, + SnapshotRecoveryAction, SnapshotRecoveryCheckout, SnapshotRecoveryCheckoutRole, + SnapshotRecoveryConfirmation, SnapshotRecoveryImage, SnapshotRecoveryImageKind, + SnapshotRecoveryPath, SnapshotRecoveryReceipt, SnapshotRecoveryResult, SnapshotRecoveryReview, + SnapshotRecoverySelection, SnapshotRecoveryState, }; pub use snapshot_work::{ SnapshotBinding, SnapshotCompletion, SnapshotNativeGuard, SnapshotNativeResult, SnapshotOrigin, diff --git a/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs index 01d12e842..f70b9c525 100644 --- a/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs +++ b/crates/cyber-server/src/runtime/snapshot_acknowledgement.rs @@ -16,10 +16,44 @@ pub struct SnapshotAcknowledgementWorker { // Actual resources are released before ordinary admission or Stop waiters. _location: Option>, checkouts: Option, + recovery: Option, _lifecycle: OwnedRwLockReadGuard<()>, _admission: Admission, _completion: Completion, } +struct RecoveryResources { + claims: super::snapshot_recovery_checkout::RecoveryClaims, + original: Option>, + resolution: Box, +} +impl RecoveryResources { + fn verify(&self, receipt: &SnapshotRecoveryReceipt) -> Result<(), String> { + use super::SnapshotRecoveryCheckoutRole as Role; + for role in [Role::Resolution, Role::Original] { + let lifetime = match role { + Role::Resolution => Some(self.resolution.as_ref()), + Role::Original => self.original.as_deref(), + }; + match (lifetime, receipt.checkout_evidence(role)) { + (Some(lifetime), Some(evidence)) => { + lifetime.verify()?; + let resources = lifetime + .checkout_bindings()? + .ok_or("Recovery checkout publication is missing")?; + if lifetime.binding() != &evidence.lifetime + || evidence.lifetime.checkout_resources_digest(&resources)? + != evidence.resources_digest + { + return Err("Recovery acknowledgement resource history changed".into()); + } + } + (None, None) => {} + _ => return Err("Recovery acknowledgement resource role is incomplete".into()), + } + } + self.claims.verify(receipt) + } +} struct RestoreResources { claims: Box, // Last: retain the original worker's lock until every native claim is released. @@ -121,6 +155,11 @@ impl SnapshotAcknowledgementWork<'_> { { resources.verify(receipt)?; } + if let (Some(resources), Acknowledgement::Recovery(receipt)) = + (&self.0.recovery, &self.0.receipt) + { + resources.verify(receipt)?; + } let authority = self.0.authority.clone(); let receipt = self.0.receipt.clone(); let origin = self.0.origin.clone(); @@ -157,7 +196,7 @@ impl SnapshotAcknowledgementWorker { let cancelled = self.control.stop.clone().drop_guard(); let joined = tokio::task::spawn_blocking(move || { work(&SnapshotAcknowledgementWork(&self))?; - if self.checkouts.is_some() { + if self.checkouts.is_some() || self.recovery.is_some() { SnapshotAcknowledgementWork(&self).authorize()?; } if let (Some(resources), Acknowledgement::Restore(receipt)) = @@ -166,6 +205,12 @@ impl SnapshotAcknowledgementWorker { resources.verify(receipt)?; resources.claims.acknowledge_restore(receipt)?; } + if let (Some(resources), Acknowledgement::Recovery(receipt)) = + (&mut self.recovery, &self.receipt) + { + resources.verify(receipt)?; + resources.claims.acknowledge(receipt)?; + } Ok(()) }); let result = joined.await.map_err(|_| { @@ -178,6 +223,72 @@ impl SnapshotAcknowledgementWorker { } impl Runtime { + async fn lock_checkout_resources( + &self, + info: &super::SessionInfo, + evidence: &super::SnapshotRestoreCheckout, + ) -> Result { + let lifetime = self + .inner + .options + .snapshots + .lock_worker(evidence.lifetime.clone()) + .await + .map_err(RuntimeError::Conflict)?; + let published = lifetime + .checkout_bindings() + .map_err(RuntimeError::Conflict)? + .ok_or_else(|| { + RuntimeError::Conflict("Original checkout publication is missing".into()) + })?; + let claims = self + .inner + .tools + .reacquire_snapshot_location(info, &evidence.lifetime, &published) + .map_err(RuntimeError::Conflict)?; + let resources = RestoreResources { claims, lifetime }; + if resources.lifetime.binding() != &evidence.lifetime { + return Err(RuntimeError::Conflict( + "Original checkout lifetime differs".into(), + )); + } + Ok(resources) + } + async fn lock_recovery_resources( + &self, + info: &super::SessionInfo, + receipt: &SnapshotRecoveryReceipt, + ) -> Result { + let checkout = receipt + .checkout + .as_ref() + .ok_or_else(|| RuntimeError::Conflict("Missing recovery checkout evidence".into()))?; + let original = match &checkout.original { + Some(original) => Some(self.lock_checkout_resources(info, original).await?), + None => None, + }; + let resolution = self + .lock_checkout_resources(info, &checkout.resolution) + .await?; + let (original_claims, original_lifetime) = match original { + Some(original) => (Some(original.claims), Some(original.lifetime)), + None => (None, None), + }; + let mut resources = RecoveryResources { + claims: super::snapshot_recovery_checkout::RecoveryClaims { + resolution: resolution.claims, + original: original_claims, + }, + original: original_lifetime, + resolution: resolution.lifetime, + }; + resources.verify(receipt).map_err(RuntimeError::Conflict)?; + resources + .claims + .prepare(receipt) + .map_err(RuntimeError::Conflict)?; + Ok(resources) + } pub(super) async fn snapshot_acknowledgement_worker( &self, handle: &Arc, @@ -260,7 +371,13 @@ impl Runtime { id, control: control.clone(), }; - let (location, checkouts) = if let Acknowledgement::Restore(restore) = &receipt + let mut recovery = None; + let (location, checkouts) = if let Acknowledgement::Recovery(receipt) = &receipt + && receipt.checkout.is_some() + { + recovery = Some(self.lock_recovery_resources(&state.info, receipt).await?); + (None, None) + } else if let Acknowledgement::Restore(restore) = &receipt && let Some(checkout) = &restore.checkout { let lifetime = self @@ -312,6 +429,7 @@ impl Runtime { control, _location: location, checkouts, + recovery, _lifecycle: lifecycle, _admission: admission, _completion: completion, @@ -321,7 +439,7 @@ impl Runtime { /// Observe both durable owner records in one read snapshot, including after a /// committed-but-lost writer reply. Filesystem completion alone never passes. -fn verify_pair( +pub(super) fn verify_pair( db: &rusqlite::Connection, receipt: &SnapshotRecoveryReceipt, origin: &SnapshotOrigin, @@ -359,5 +477,48 @@ fn verify_pair( "Journal acknowledgement requires the exact committed native pair".into(), )); } + if let Some(checkout) = &receipt.checkout { + receipt + .validate_checkout(origin) + .map_err(cyber_store::StoreError::InvalidAppend)?; + if resolution.snapshot_lifetime.as_ref() != Some(&checkout.resolution.lifetime) + || original + .snapshot_lifetime + .as_ref() + .filter(|life| life.version == 2) + != checkout + .original + .as_ref() + .map(|evidence| &evidence.lifetime) + { + return Err(cyber_store::StoreError::InvalidAppend( + "Recovery checkout lifetime history differs".into(), + )); + } + } Ok(()) } + +pub(super) fn verify_committed( + db: &rusqlite::Connection, + receipt: &SnapshotRecoveryReceipt, +) -> Result<(), cyber_store::StoreError> { + let owner = super::snapshot_recovery::request_owner( + &receipt.original.session_created_id, + &receipt.request_id, + ); + let data: String = db.query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq DESC LIMIT 1", + [&owner, activity::CHANGED], + |row| row.get(0), + )?; + let record: activity::Record = serde_json::from_str(&data) + .map_err(|error| cyber_store::StoreError::InvalidAppend(error.to_string()))?; + verify_pair( + db, + receipt, + record.snapshot_origin.as_ref().ok_or_else(|| { + cyber_store::StoreError::InvalidAppend("Recovery resolution origin is missing".into()) + })?, + ) +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index c177cd843..e98225a8e 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -208,6 +208,15 @@ impl Runtime { )); } }; + if let Some(receipt) = &record.snapshot_receipt { + let receipt = receipt.clone(); + self.inner.store.read(move |db| { + let tx = db.unchecked_transaction()?; + super::snapshot_acknowledgement::verify_committed(&tx, &receipt)?; + tx.commit()?; + Ok(()) + })?; + } Ok(Some(super::SnapshotRecoveryResult { request: request.confirmation, original: request.original, diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_checkout.rs b/crates/cyber-server/src/runtime/snapshot_recovery_checkout.rs new file mode 100644 index 000000000..0215dd94e --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_recovery_checkout.rs @@ -0,0 +1,126 @@ +//! Runtime-owned checkout evidence, distinct from immutable native intent data. +use super::{ + SnapshotOrigin, SnapshotRecoveryCheckoutRole as Role, SnapshotRecoveryReceipt as Receipt, + SnapshotRestoreCheckout, +}; + +impl Receipt { + pub fn checkout_evidence(&self, role: Role) -> Option<&SnapshotRestoreCheckout> { + let checkout = self.checkout.as_ref()?; + match role { + Role::Original => checkout.original.as_deref(), + Role::Resolution => Some(&checkout.resolution), + } + } + /// A checksum binds metadata to an observed durable pair; it grants no authority alone. + pub fn checkout_acknowledgement_digest(&self) -> Result { + use sha2::{Digest, Sha256}; + let checkout = self + .checkout + .as_ref() + .ok_or("Legacy recovery receipt lacks checkout proof")?; + self.validate_checkout(&checkout.resolution_origin)?; + Ok(format!( + "{:x}", + Sha256::digest( + serde_json::to_vec(&("snapshot-recovery-checkout-ack-v1", self)) + .map_err(|error| error.to_string())? + ) + )) + } + pub(super) fn validate_checkout(&self, resolution: &SnapshotOrigin) -> Result<(), String> { + let checkout = self + .checkout + .as_ref() + .ok_or("Missing recovery checkout proof")?; + let hash = + |value: &str| value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()); + if checkout.version != 1 + || checkout.resolution_origin != *resolution + || !self.filesystem_complete + || self + .resolution_digest + .as_deref() + .is_none_or(|value| !hash(value)) + || !hash(&self.confirmation) + || self.request_id.is_empty() + || self.request_id.len() > 256 + || self.request_id.contains('\0') + || self.journal_id != self.original.binding.journal_id + || self.resolution_id != resolution.binding.journal_id + || resolution.activity_id + != super::snapshot_recovery::request_owner( + &self.original.session_created_id, + &self.request_id, + ) + || resolution.activity_id == self.original.activity_id + || self.resolution_id == self.journal_id + || resolution.session_id != self.original.session_id + || resolution.session_created_id != self.original.session_created_id + || resolution.worktree_id != self.original.worktree_id + || resolution.binding.directory != self.original.binding.directory + || resolution.binding.root_identity != self.original.binding.root_identity + { + return Err("Recovery checkout receipt owner or frame differs".into()); + } + checkout.resolution.validate(resolution)?; + if let Some(original) = &checkout.original { + original.validate(&self.original)?; + } + Ok(()) + } +} + +pub(super) struct RecoveryClaims { + pub resolution: Box, + pub original: Option>, +} +impl RecoveryClaims { + pub fn verify(&self, receipt: &Receipt) -> Result<(), String> { + self.resolution.verify()?; + if let Some(original) = &self.original { + original.verify()?; + } + if receipt.checkout.is_none() { + return Ok(()); + } + for role in [Role::Resolution, Role::Original] { + let claims = match role { + Role::Resolution => Some(&self.resolution), + Role::Original => self.original.as_ref(), + }; + match (receipt.checkout_evidence(role), claims) { + (Some(evidence), Some(claims)) + if evidence + .lifetime + .checkout_resources_digest(&claims.bindings()?)? + == evidence.resources_digest => {} + (None, None) => {} + _ => return Err("Recovery consuming checkout set differs".into()), + } + } + Ok(()) + } + pub fn prepare(&mut self, receipt: &Receipt) -> Result<(), String> { + self.verify(receipt)?; + if receipt.checkout.is_some() { + self.resolution + .prepare_recovery_acknowledgement(receipt, Role::Resolution)?; + if let Some(original) = &mut self.original { + original.prepare_recovery_acknowledgement(receipt, Role::Original)?; + } + } + Ok(()) + } + pub fn acknowledge(&mut self, receipt: &Receipt) -> Result<(), String> { + self.verify(receipt)?; + if receipt.checkout.is_some() { + self.resolution + .acknowledge_recovery(receipt, Role::Resolution)?; + if let Some(original) = &mut self.original { + original.acknowledge_recovery(receipt, Role::Original)?; + } + } + Ok(()) + } +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs index ad39415e7..839a203ac 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery_types.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery_types.rs @@ -96,6 +96,25 @@ pub struct SnapshotRecoveryReceipt { /// Missing on older receipts; those cannot authorize new journal writes. #[serde(default, skip_serializing_if = "Option::is_none")] pub resolution_digest: Option, + /// Runtime-attested original and resolution resources; absent on legacy receipts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkout: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SnapshotRecoveryCheckout { + pub version: u32, + pub resolution_origin: super::SnapshotOrigin, + /// None preserves missing legacy proof, rather than claiming an empty set. + pub original: Option>, + pub resolution: Box, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SnapshotRecoveryCheckoutRole { + Original, + Resolution, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 9a955b8b8..33b349430 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -135,7 +135,7 @@ pub struct SnapshotCompletion { struct OriginalResources { origin: SnapshotOrigin, - claims: Box, + claims: Option>, bindings: Vec, // Last: keep the exited original's lock through all native claim disposal. lifetime: Box, @@ -146,8 +146,12 @@ impl OriginalResources { if self.lifetime.checkout_bindings()?.as_ref() != Some(&self.bindings) { return Err("Original recovery checkout publication changed".into()); } - self.claims.verify()?; - if self.claims.bindings()? != self.bindings { + let claims = self + .claims + .as_ref() + .ok_or("Original recovery claims are not retained")?; + claims.verify()?; + if claims.bindings()? != self.bindings { return Err("Original recovery checkout ownership changed".into()); } super::SnapshotRestoreCheckout { @@ -386,7 +390,7 @@ impl SnapshotCompletion { mut self, state: &SessionState, ) -> Result { - let receipt = self + let mut receipt = self .native .recovery .as_ref() @@ -399,18 +403,82 @@ impl SnapshotCompletion { if let Some(original) = &self.worker._original { original.verify().map_err(RuntimeError::Conflict)?; } + if let Some(lifetime) = &self.worker._lifetime { + lifetime.verify().map_err(RuntimeError::Conflict)?; + let published = lifetime + .checkout_bindings() + .map_err(RuntimeError::Conflict)? + .ok_or_else(|| { + RuntimeError::Conflict("Resolution resources are unpublished".into()) + })?; + let claims = self.worker.location().map_err(RuntimeError::Conflict)?; + if claims.bindings().map_err(RuntimeError::Conflict)? != published { + return Err(RuntimeError::Conflict( + "Resolution checkout publication differs".into(), + )); + } + let resolution = super::SnapshotRestoreCheckout { + version: 1, + lifetime: lifetime.binding().clone(), + resources_digest: lifetime + .binding() + .checkout_resources_digest(&published) + .map_err(RuntimeError::Conflict)?, + }; + let original = self + .worker + ._original + .as_ref() + .map(|original| { + Ok::<_, RuntimeError>(Box::new(super::SnapshotRestoreCheckout { + version: 1, + lifetime: original.lifetime.binding().clone(), + resources_digest: original + .lifetime + .binding() + .checkout_resources_digest(&original.bindings) + .map_err(RuntimeError::Conflict)?, + })) + }) + .transpose()?; + let resolution_origin = self + .worker + .scope + .snapshot_origin() + .cloned() + .ok_or_else(|| RuntimeError::Conflict("Resolution has no native origin".into()))?; + receipt.checkout = Some(Box::new(super::SnapshotRecoveryCheckout { + version: 1, + resolution_origin: resolution_origin.clone(), + original, + resolution: Box::new(resolution), + })); + receipt + .validate_checkout(&resolution_origin) + .map_err(RuntimeError::Conflict)?; + } let location = self .worker ._location .take() .ok_or_else(|| RuntimeError::Conflict("Missing snapshot checkout proof".into()))?; location.verify().map_err(RuntimeError::Conflict)?; - self.worker._location = Some(self.worker.scope.commit_recovery( - state, - self.worker.sequence, - &receipt, - location, - )?); + let claims = super::snapshot_recovery_checkout::RecoveryClaims { + resolution: location, + original: self + .worker + ._original + .as_mut() + .and_then(|original| original.claims.take()), + }; + let claims = + self.worker + .scope + .commit_recovery(state, self.worker.sequence, &receipt, claims)?; + self.worker._location = Some(claims.resolution); + if let Some(original) = &mut self.worker._original { + original.claims = claims.original; + } self.native.guard.acknowledge(self.worker.scope.snapshot_origin()).map_err(|error| RuntimeError::Conflict(format!( "Recovery runtime acknowledgement committed; native journal acknowledgement requires reconciliation: {error}" )))?; @@ -553,7 +621,7 @@ impl Runtime { .map_err(RuntimeError::Conflict)?; let original = OriginalResources { origin: request.original.clone(), - claims, + claims: Some(claims), bindings, lifetime, }; @@ -563,6 +631,11 @@ impl Runtime { None }; let lifetime = self.snapshot_lifetime(state, &id).await?; + if original.is_some() && lifetime.is_none() { + return Err(RuntimeError::Conflict( + "Resolution lifetime proof is unavailable".into(), + )); + } let scope = activity::Scope::reserve_recovery( self, handle.clone(), diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 0ff297863..1505876b6 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -2039,6 +2039,7 @@ async fn finish_recovery( original: request.original, filesystem_complete: true, resolution_digest: Some("d".repeat(64)), + checkout: None, }); Ok(native) }) @@ -2589,6 +2590,7 @@ async fn ordinary_restore_commit_cannot_consume_a_recovery_receipt_or_acknowledg original: origin, filesystem_complete: true, resolution_digest: Some("d".repeat(64)), + checkout: None, }); Ok(result) }) @@ -3530,3 +3532,250 @@ async fn native_result_cleanup_precedes_worker_lifetime_exit() { assert_eq!(permits.available_permits(), 1); runtime.shutdown().await; } + +struct RecoveryRoleAck { + role: super::super::SnapshotRecoveryCheckoutRole, + fail: &'static str, + order: Arc>>, +} +impl super::super::SnapshotLocationGuard for RecoveryRoleAck { + fn verify(&self) -> Result<(), String> { + Ok(()) + } + fn prepare_recovery_acknowledgement( + &mut self, + receipt: &super::super::SnapshotRecoveryReceipt, + role: super::super::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + assert_eq!(role, self.role); + receipt.checkout_acknowledgement_digest()?; + self.order.lock().unwrap().push(format!("prepare {role:?}")); + if self.fail == "candidate" && role == super::super::SnapshotRecoveryCheckoutRole::Original + { + Err("fixture rejected original candidate".into()) + } else { + Ok(()) + } + } + fn acknowledge_recovery( + &mut self, + receipt: &super::super::SnapshotRecoveryReceipt, + role: super::super::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + assert_eq!(role, self.role); + receipt.checkout_acknowledgement_digest()?; + self.order.lock().unwrap().push(format!("publish {role:?}")); + if self.fail == "publication" + && role == super::super::SnapshotRecoveryCheckoutRole::Original + { + Err("fixture interrupted original publication".into()) + } else { + Ok(()) + } + } +} + +#[tokio::test] +async fn recovery_checkout_pair_binds_both_roles_and_reconciles_without_replaying_source() { + use super::super::SnapshotRecoveryCheckoutRole as Role; + fn lost_reply(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("recovery checkout pair reply lost".into()), + )) + } + for fail in ["valid", "candidate", "publication", "reply"] { + let (temp, base, _, _, _) = super::super::binary_documents_tests::fixture().await; + let store = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(temp.path().join("checkout-pair.db")), + Runtime::registry(), + )) + .unwrap(); + let permits = Arc::new(tokio::sync::Semaphore::new(2)); + let resources = Arc::new(std::sync::Mutex::new(None)); + let metadata = Arc::new(AtomicUsize::new(0)); + let runtime = runtime_with_snapshots( + &base, + Arc::new(store), + temp.path(), + Arc::new(Lifetimes { + inner: Acknowledgements(metadata.clone()), + permits: permits.clone(), + foreign: false, + reject_resources: false, + resources: resources.clone(), + }), + ); + let session = runtime + .create_session(super::super::CreateSession { + directory: temp.path().to_str().unwrap().into(), + model: "test/main".into(), + ..Default::default() + }) + .await + .unwrap() + .id; + let original = + failed_snapshot(&runtime, &session, temp.path().canonicalize().unwrap()).await; + let request = recovery_request(original.clone()); + let handle = runtime.inner.handle(&session).await.unwrap(); + let state = handle.state.lock().await; + let mut owner = runtime + .snapshot_recovery_worker(&handle, &state, request.clone()) + .await + .unwrap(); + drop(state); + let resolution_activity = owner.scope.id().to_owned(); + let order = Arc::new(std::sync::Mutex::new(vec![])); + owner._location = Some(Box::new(RecoveryRoleAck { + role: Role::Resolution, + fail, + order: order.clone(), + })); + owner._original.as_mut().unwrap().claims = Some(Box::new(RecoveryRoleAck { + role: Role::Original, + fail, + order: order.clone(), + })); + let journal_acks = Arc::new(AtomicUsize::new(0)); + let completed = finish_recovery(owner, request.clone(), journal_acks.clone()).await; + if fail == "reply" { + runtime + .inner + .store + .transaction(|tx| { + tx.wal_hook(Some(lost_reply)); + Ok(()) + }) + .unwrap(); + } + let state = handle.state.lock().await; + let result = completed.commit_recovery(&state); + drop(state); + assert_eq!(permits.available_permits(), 2); + let lookup = runtime + .snapshot_recovery_lookup(&session, &request.confirmation.request_id) + .await + .unwrap() + .unwrap(); + if fail == "candidate" { + assert!(result.is_err()); + assert!(lookup.receipt.is_none()); + assert_eq!( + *order.lock().unwrap(), + ["prepare Resolution", "prepare Original"] + ); + assert_eq!(journal_acks.load(Ordering::SeqCst), 0); + } else { + if fail == "reply" { + assert!(result.is_err()); + } else if fail == "publication" { + assert!( + result + .unwrap_err() + .to_string() + .contains("runtime acknowledgement committed") + ); + } else { + result.unwrap(); + } + let receipt = lookup.receipt.unwrap(); + let checkout = receipt.checkout.as_ref().unwrap(); + assert_eq!( + checkout + .original + .as_ref() + .unwrap() + .lifetime + .owner + .activity_id, + original.activity_id + ); + assert_eq!( + checkout.resolution.lifetime.owner.activity_id, + resolution_activity + ); + assert_eq!(checkout.resolution_origin.activity_id, resolution_activity); + let mut expected_order = vec!["prepare Resolution", "prepare Original"]; + if fail != "reply" { + expected_order.extend(["publish Resolution", "publish Original"]); + } + assert_eq!(*order.lock().unwrap(), expected_order); + let hash = receipt.checkout_acknowledgement_digest().unwrap(); + let mut changed = receipt.clone(); + changed + .checkout + .as_mut() + .unwrap() + .original + .as_mut() + .unwrap() + .resources_digest = "f".repeat(64); + assert_ne!(changed.checkout_acknowledgement_digest().unwrap(), hash); + assert!( + runtime + .inner + .store + .read( + move |db| super::super::snapshot_acknowledgement::verify_committed( + db, &changed + ) + ) + .is_err() + ); + if fail == "publication" || fail == "reply" { + *resources.lock().unwrap() = None; + assert!( + runtime + .snapshot_recovery_acknowledge(&session, &request.confirmation.request_id) + .await + .is_err() + ); + assert_eq!(metadata.load(Ordering::SeqCst), 0); + *resources.lock().unwrap() = Some(vec![]); + for _ in 0..2 { + let reconciled = runtime + .snapshot_recovery_acknowledge(&session, &request.confirmation.request_id) + .await + .unwrap(); + assert_eq!(reconciled.receipt.as_ref(), Some(&receipt)); + assert_eq!(reconciled.journal_acknowledged, Some(true)); + } + assert_eq!(metadata.load(Ordering::SeqCst), 2); + assert_eq!(journal_acks.load(Ordering::SeqCst), 0); + } + if fail == "reply" { + runtime.shutdown().await; + let reopened = cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(temp.path().join("checkout-pair.db")), + Runtime::registry(), + )) + .unwrap(); + let reloaded = runtime_with_snapshots( + &base, + Arc::new(reopened), + temp.path(), + runtime.inner.options.snapshots.clone(), + ); + let observed = reloaded + .snapshot_recovery_acknowledge(&session, &request.confirmation.request_id) + .await + .unwrap(); + assert_eq!(observed.receipt.as_ref(), Some(&receipt)); + assert_eq!(metadata.load(Ordering::SeqCst), 3); + assert_eq!(journal_acks.load(Ordering::SeqCst), 0); + reloaded.shutdown().await; + } + let mut legacy = receipt; + legacy.checkout = None; + assert!( + serde_json::to_value(&legacy) + .unwrap() + .get("checkout") + .is_none() + ); + assert!(legacy.checkout_acknowledgement_digest().is_err()); + } + runtime.shutdown().await; + } +} diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs index 284daaba1..32142d361 100644 --- a/crates/cyber-snapshot/src/copy/resolution.rs +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -243,7 +243,11 @@ impl Store { if expected.resolution_digest.is_none() { observed.resolution_digest = None; } - if observed != *expected { + // Checkout evidence is consumed and attested by the runtime writer, not + // fabricated from native journal bytes. This compares native intent only. + let mut expected = expected.clone(); + expected.checkout = None; + if observed != expected { return Err("Native resolution receipt changed".into()); } Ok(()) @@ -733,6 +737,7 @@ fn receipt(id: &str, record: &Journal) -> Result { original: resolution.original.clone(), filesystem_complete: true, resolution_digest: Some(immutable_digest(record)?), + checkout: None, }) } diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index d0e396db3..182f3161a 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -1322,9 +1322,58 @@ impl cyber_server::runtime::SnapshotLocationGuard for ActiveSnapshotLocation { ) -> Result<(), String> { self.proof.acknowledge_restore(receipt) } + fn prepare_recovery_acknowledgement( + &mut self, + receipt: &cyber_server::runtime::SnapshotRecoveryReceipt, + role: cyber_server::runtime::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + self.proof.prepare_recovery_acknowledgement(receipt, role) + } + fn acknowledge_recovery( + &mut self, + receipt: &cyber_server::runtime::SnapshotRecoveryReceipt, + role: cyber_server::runtime::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + self.proof.acknowledge_recovery(receipt, role) + } } struct RetainedSnapshotProof(Vec); +impl RetainedSnapshotProof { + fn recovery_acknowledgement( + &mut self, + receipt: &cyber_server::runtime::SnapshotRecoveryReceipt, + role: cyber_server::runtime::SnapshotRecoveryCheckoutRole, + publish: bool, + ) -> Result<(), String> { + use cyber_server::runtime::SnapshotLocationGuard; + let evidence = receipt + .checkout_evidence(role) + .ok_or("Legacy recovery receipt lacks resource role")?; + let digest = evidence + .lifetime + .checkout_resources_digest(&self.bindings()?)?; + if digest != evidence.resources_digest { + return Err("Recovery resource set differs".into()); + } + let receipt_digest = receipt.checkout_acknowledgement_digest()?; + for claim in &mut self.0 { + let ack = cyber_core::worktrees::CheckoutAcknowledgement::new( + claim.binding(), + &digest, + &receipt_digest, + ) + .map_err(|error| error.to_string())?; + if publish { + ack.publish(claim) + } else { + ack.retain_existing(claim) + } + .map_err(|error| error.to_string())?; + } + self.verify() + } +} impl cyber_server::runtime::SnapshotLocationGuard for RetainedSnapshotProof { fn verify(&self) -> Result<(), String> { self.0 @@ -1429,6 +1478,20 @@ impl cyber_server::runtime::SnapshotLocationGuard for RetainedSnapshotProof { } self.verify() } + fn prepare_recovery_acknowledgement( + &mut self, + receipt: &cyber_server::runtime::SnapshotRecoveryReceipt, + role: cyber_server::runtime::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + self.recovery_acknowledgement(receipt, role, false) + } + fn acknowledge_recovery( + &mut self, + receipt: &cyber_server::runtime::SnapshotRecoveryReceipt, + role: cyber_server::runtime::SnapshotRecoveryCheckoutRole, + ) -> Result<(), String> { + self.recovery_acknowledgement(receipt, role, true) + } } impl cyber_server::runtime::LocationGuard for WorktreeLease { diff --git a/crates/cyber-tools/tests/worktree_setup.rs b/crates/cyber-tools/tests/worktree_setup.rs index b939a7b73..62b50df65 100644 --- a/crates/cyber-tools/tests/worktree_setup.rs +++ b/crates/cyber-tools/tests/worktree_setup.rs @@ -3208,3 +3208,181 @@ async fn cwd_changed_hook_shutdown_releases_outer_worktree_admission() { assert_eq!(events.len(), 1); assert_cwd_transition(&flow, &id, &events[0]); } + +#[tokio::test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn recovery_checkout_roles_publish_exact_markers_without_rewriting_original_claims() { + use cyber_server::runtime::{ + CreateSession, NoSnapshots, SnapshotBinding, SnapshotLifetimeBinding, + SnapshotLifetimeRequest, SnapshotOrigin, SnapshotRecoveryAction, SnapshotRecoveryCheckout, + SnapshotRecoveryCheckoutRole as Role, SnapshotRecoveryReceipt, SnapshotRestoreCheckout, + ToolHost, + }; + use sha2::{Digest, Sha256}; + let (fixture, _, managed) = owned().await; + let flow = support::flow::Flow::with(fixture, vec![], false, Arc::new(NoSnapshots)); + let info = flow + .runtime + .create_session(CreateSession { + directory: managed.path.display().to_string(), + model: "test/main".into(), + mode: Some("bypass".into()), + ..Default::default() + }) + .await + .unwrap(); + let created = flow.f.store.read_events(&info.id, -1, 1).unwrap().events[0] + .id + .clone(); + let request = "fixture-recovery-roles"; + // Construct the protocol-owned fixture ID; runtime tests attest the actual pair. + let mut hash = Sha256::new(); + hash.update(b"snapshot-recovery-request-v1"); + hash.update((created.len() as u64).to_be_bytes()); + hash.update(created.as_bytes()); + hash.update((request.len() as u64).to_be_bytes()); + hash.update(request.as_bytes()); + let resolution_owner = format!("op_snapshot_recovery_{:x}", hash.finalize()); + let lifetime = |activity: &str| SnapshotLifetimeBinding { + version: 2, + owner: SnapshotLifetimeRequest { + activity_id: activity.into(), + session_id: info.id.clone(), + session_created_id: created.clone(), + directory: managed.path.clone(), + worktree_id: Some(managed.id.clone()), + }, + root_identity: "fixture-root".into(), + namespace_identity: "fixture-location-namespace".into(), + directory_identity: "fixture-workers".into(), + file_identity: format!("fixture-{activity}"), + source_phase: Some(format!("phase_{activity}")), + }; + let original_lifetime = lifetime("op_original_roles"); + let resolution_lifetime = lifetime(&resolution_owner); + let origin = |life: &SnapshotLifetimeBinding, journal: &str| SnapshotOrigin { + version: 1, + session_id: info.id.clone(), + session_created_id: created.clone(), + activity_id: life.owner.activity_id.clone(), + worktree_id: Some(managed.id.clone()), + binding: SnapshotBinding { + directory: managed.path.clone(), + root_identity: life.root_identity.clone(), + namespace_identity: "fixture-journals".into(), + journal_id: journal.into(), + journal_identity: format!("fixture-{journal}"), + lock_identity: Some("fixture-lock".into()), + }, + }; + let original = flow + .f + .host + .claim_snapshot_location_bound( + &info, + "op_original_roles", + Some(&original_lifetime), + CancellationToken::new(), + ) + .await + .unwrap(); + let resolution = flow + .f + .host + .claim_snapshot_location_bound( + &info, + &resolution_owner, + Some(&resolution_lifetime), + CancellationToken::new(), + ) + .await + .unwrap(); + let original_set = original.snapshot_bindings().unwrap(); + let resolution_set = resolution.snapshot_bindings().unwrap(); + let directory = managed + .common_dir + .join("cyber-worktree-activity") + .join(&managed.id); + let original_path = directory.join("op_original_roles.lock"); + let resolution_path = directory.join(format!("{resolution_owner}.lock")); + let original_bytes = std::fs::read(&original_path).unwrap(); + let resolution_bytes = std::fs::read(&resolution_path).unwrap(); + let mut original = original.settle_snapshot_retained().unwrap(); + let mut resolution = resolution.settle_snapshot_retained().unwrap(); + let receipt = SnapshotRecoveryReceipt { + journal_id: "restore_original_roles".into(), + resolution_id: "restore_resolution_roles".into(), + request_id: request.into(), + action: SnapshotRecoveryAction::Rollback, + confirmation: "a".repeat(64), + original: origin(&original_lifetime, "restore_original_roles"), + filesystem_complete: true, + resolution_digest: Some("b".repeat(64)), + checkout: Some(Box::new(SnapshotRecoveryCheckout { + version: 1, + resolution_origin: origin(&resolution_lifetime, "restore_resolution_roles"), + original: Some(Box::new(SnapshotRestoreCheckout { + version: 1, + lifetime: original_lifetime.clone(), + resources_digest: original_lifetime + .checkout_resources_digest(&original_set) + .unwrap(), + })), + resolution: Box::new(SnapshotRestoreCheckout { + version: 1, + lifetime: resolution_lifetime.clone(), + resources_digest: resolution_lifetime + .checkout_resources_digest(&resolution_set) + .unwrap(), + }), + })), + }; + original + .prepare_recovery_acknowledgement(&receipt, Role::Original) + .unwrap(); + resolution + .prepare_recovery_acknowledgement(&receipt, Role::Resolution) + .unwrap(); + assert!(!directory.join("op_original_roles.snapshot-ack").exists()); + assert!( + original + .acknowledge_recovery(&receipt, Role::Resolution) + .is_err() + ); + resolution + .acknowledge_recovery(&receipt, Role::Resolution) + .unwrap(); + original + .acknowledge_recovery(&receipt, Role::Original) + .unwrap(); + let marker = directory.join("op_original_roles.snapshot-ack"); + let before = std::fs::read(&marker).unwrap(); + original + .acknowledge_recovery(&receipt, Role::Original) + .unwrap(); + assert_eq!(std::fs::read(&marker).unwrap(), before); + let data: serde_json::Value = serde_json::from_slice(&before).unwrap(); + assert_eq!( + data["receipt_digest"], + receipt.checkout_acknowledgement_digest().unwrap() + ); + let mut foreign = receipt.clone(); + foreign.action = SnapshotRecoveryAction::Keep; + assert!( + original + .prepare_recovery_acknowledgement(&foreign, Role::Original) + .is_err() + ); + let mut legacy = receipt; + legacy.checkout = None; + assert!( + original + .acknowledge_recovery(&legacy, Role::Original) + .is_err() + ); + drop(original); + drop(resolution); + assert_eq!(std::fs::read(original_path).unwrap(), original_bytes); + assert_eq!(std::fs::read(resolution_path).unwrap(), resolution_bytes); + flow.runtime.shutdown().await; +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index bf2d00e5d..0ff731bbe 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -207,3 +207,13 @@ reconciliation. Acknowledged creation metadata is retained across authorized worktree removal/recreation. Partial acquisition/corrupt construction recovery, post-effect/owned Git integration, reference-aware reclamation and full native/ client acceptance remain open; this does not complete the recovery delivery increment. + + +The recovery consuming-receipt continuation binds both retained checkout owners to +the durable pair and publishes exact resolution-then-original native disposition +markers only after observing commit. Explicit reconciliation reacquires both +published resource sets without source replay; original claim bytes and legacy +unknown fences remain protected. This is local implementation progress, not full +recovery delivery: actual managed Git source callbacks, partial construction, +reference-aware reclamation and complete native/platform/client acceptance remain +open. See the daily coding implementation evidence and authoritative parent tasks. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 41f128b37..3a205a3b6 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2386,3 +2386,91 @@ passed. Pinned strict OpenSpec passed all 117 items; spec lint reported 0 errors cross-process retention, real TUI/App, Linux/Windows and source-bound CI were not run for this increment. Consuming recovery resource evidence, exact native marker publication/reconciliation and reference-aware cleanup remain open. + + +## Recovery checkout receipt and native disposition continuation (2026-11-07) + +Recovery now enriches the native resolution receipt with runtime-attested checkout +evidence: exact resolution origin, its version-2 lifetime and complete resource-set +digest, plus the original version-2 lifetime/resource-set digest when available. +Missing legacy original proof remains absent. The runtime verifies the immutable +native receipt before enrichment; native journal verification compares native +intent fields only, while Store verification attests the added runtime evidence. +The receipt checksum covers both kinds of evidence. It is a disposition binding, +not independent acknowledgement or source admission authority. + +The consuming paired writer verifies both retained claim sets and proposed markers +read-only, checks exact original/resolution lifetime history, and refuses missing +version-2 evidence. After observing the durable pair, it publishes resolution +markers before original markers. Exact original claim bytes remain unchanged. +Failure after commit leaves the durable pair available to lookup and explicit +reconciliation. Reconciliation first verifies the committed pair, locks the exact +original and resolution lifetimes, reacquires their complete published resource +sets, and performs bounded metadata work without invoking source restore. Resource +or marker changes refuse. Existing matching markers are idempotent. + +The code-review skill was applied across runtime ports/writer, native journal and +host adapters, tests, schema/SDK and spec changes. Review covered independent +owner identity, no legacy adoption, original-before-resolution lock acquisition, +claim-before-lifetime disposal, read-only candidate verification before the atomic +pair, publication ordering, exact committed-pair lookup, and retry without source +replay. The runtime regression uses actual Store operations with instrumented +lifetime/Location ports; the native host regression uses actual managed checkout +claims/pins/markers with fixture lifetime/receipt context. Neither alone proves +the full managed Git runtime path. Real CLI recovery fixtures use unmanaged +Locations and verify receipt/native lifetime correspondence. + +Actual managed Git source admission remains disabled. Partial-constructor +recovery, reference/reader/lease-aware reclamation, complete native cuts and +platform/client acceptance remain required. No parent task or either issue is +completed by this continuation. + + +Local macOS arm64 verification for this continuation: + +- Locked server library tests: 435 passed, 1 ignored. The new recovery case + covers candidate refusal, successful paired disposition, interrupted original + marker publication, committed-but-lost WAL reply, read-only lookup, missing + publication refusal, repeated metadata reconciliation and reopened Store retry. + The lost-reply test initially attempted a prohibited mutation through the + failed Store; it now uses read-only lookup/reconciliation and explicitly opens + the same database again, preserving production Store failure semantics. +- Locked snapshot suite: 137 passed, 2 ignored across unit/integration binaries; + doc-test binaries had no tests. Native intent comparison and legacy receipts + retain their existing behavior without treating runtime checkout context as + journal-supplied evidence. +- Locked complete native worktree suite: 41 passed. The added managed-claim test + checks read-only candidate preparation, wrong-role refusal, exact two-role + marker publication, idempotence, changed receipt refusal, legacy refusal and + byte-for-byte preservation of both original claim files. +- Locked real CLI branch/recovery suite: all 17 passed with new assertions for + both receipt owners and actual worker lifetime/source-phase publications. + Existing partial restore kills, keep/rollback, restart/retry, code/conversation + stage/clear and no-op cases executed. These Locations are unmanaged; this does + not accept managed Git source integration. +- OpenAPI regeneration test passed. Generated TypeScript consistency and separate + SDK typechecking passed. SDK tests with CYBER_BIN pointing at the freshly built + CLI passed 142 cases, with 2 Linux-only protected-file/vault cases skipped on + macOS. The earlier run without CYBER_BIN passed 134 with 10 skipped. +- Formatting and diff checks passed. Pinned strict OpenSpec passed 117 items; + spec lint reported 0 errors and 25 warnings, without baseline attribution. + +Native C compilation emitted sandbox_init/sandbox_free_error deprecation warnings; +no unchanged-main comparison or pre-existing attribution was made. Full workspace +execution, Linux/Windows native runs, real TUI/App acceptance and source-bound CI +were not performed for this continuation. Full recovery/cleanup and both issues +remain open. + + +Four-package all-target locked Clippy (server/snapshot/tools/CLI) passed with Rust +warnings denied after correcting an Option source-phase path in the CLI assertion +helper. Server all-target locked Clippy passed again after the lost-reply test was +corrected. No warning allowances or production Store recovery changes were added. + +Review verdict: approve this local consuming-receipt continuation. Critical: 0; +major: 0; minor: 0; nits: 0 unresolved findings. The changes preserve the existing +runtime/host port boundary and native adapter ownership. Executable Cargo unit, +integration/native and real CLI tests cover the added behavior; generated SDK +checks and real-client tests pass as recorded above. This review does not approve +the full recovery delivery increment or close the explicitly retained managed Git, +partial-construction, reclamation, crash/platform/client and CI acceptance gaps. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index d9cf16e24..fdff43612 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -616,3 +616,13 @@ foreign or changed proof refuses before resolution dispatch. Legacy owners SHALL NOT acquire retrospective checkout proof. Completion disposal SHALL release native result guards before the worker's lifetime locks so guard cleanup cannot outlive the native ownership proof. Retention alone grants no checkout acknowledgement. + +New consuming recovery receipts SHALL bind the resolution origin and its complete +version-2 checkout evidence, plus original evidence whenever the interrupted owner +has version-2 proof. Both native claim sets SHALL be verified at the paired writer. +Only an observed durable pair may publish matching resolution markers followed by +original markers. The checksum SHALL cover both roles and the immutable native +resolution digest. Native journal verification remains responsible for its own +immutable intent; runtime checkout evidence is additionally attested by the writer. +Legacy receipts omit the section and gain no marker authority. Missing original +version-2 proof cannot be silently downgraded to legacy evidence. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 85266291a..28f4e729a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -298,6 +298,17 @@ - **AND** source authorization and consuming completion SHALL revalidate original resources; legacy owners SHALL gain no retrospective proof and unknown lifetime versions SHALL refuse - **AND** retaining owners alone SHALL NOT acknowledge either native activity, and native result guard disposal SHALL precede worker lifetime release +#### Scenario: Recovery consumes both checkout owners before native disposition +- **WHEN** recovery finishes with an exact version-2 resolution lifetime and an attested version-2 original lifetime +- **THEN** its consuming receipt SHALL bind the resolution origin and both complete checkout-set digests, with both retained claim sets checked read-only and both lifetime histories checked at the paired writer +- **AND** observing the durable pair SHALL precede resolution checkout acknowledgement followed by original checkout acknowledgement, preserving original claim bytes and binding markers to the complete recovery receipt +- **AND** an original without version-2 lifetime proof SHALL remain explicitly absent rather than being adopted or inferred as an empty checkout set + +#### Scenario: Recovery checkout acknowledgement resumes after an uncertain reply +- **WHEN** a consuming recovery pair commits but its reply or subsequent native marker publication fails +- **THEN** lookup SHALL verify the exact durable pair without replaying source work, and explicit reconciliation SHALL reacquire the original and resolution lifetimes and complete published checkout sets before bounded metadata acknowledgement +- **AND** changed, missing or live resource proof SHALL refuse reconciliation, matching markers SHALL be idempotent, and receipts without checkout evidence SHALL gain no new native claim acknowledgement authority + ## MODIFIED Requirements ### Requirement: Non-git fallback diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 0b14d7a89..62f057baa 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -391,3 +391,17 @@ worker lifetime locks. This is retention, not original/resolution checkout marke authority: binding both sets into consuming recovery receipts and publishing/ reconciling their exact acknowledgements remain required. M2–M8 and parent tasks stay open; ordinary Git source admission is unchanged and disabled. + + +Recovery checkout consuming-receipt continuation: new recovery receipts bind the +exact resolution origin, resolution lifetime/resource digest, and the original +version-2 lifetime/resource digest when available. Both retained claim sets are +checked read-only at the consuming paired writer, which checks both attested +lifetime histories and refuses a missing version-2 proof. Only an observed durable +pair admits exact resolution-then-original native checkout markers. Original +claim bytes stay immutable. Lookup verifies both committed owners, and explicit +metadata reconciliation reacquires both original resource sets rather than +substituting a fresh Session lease or replaying source work. Legacy receipts gain +no new claim authority. This continuation does not close M2–M8 or parent tasks: +partial acquisition, actual managed Git source integration, reference-aware +reclamation and full native/platform/client acceptance remain required. diff --git a/sdk/openapi.json b/sdk/openapi.json index f1a79158e..43478425a 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7485,6 +7485,17 @@ "null" ], "description": "Immutable native resolution data, excluding acknowledgement/progress.\nMissing on older receipts; those cannot authorize new journal writes." + }, + "checkout": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRecoveryCheckout" + }, + { + "type": "null" + } + ], + "description": "Runtime-attested original and resolution resources; absent on legacy receipts." } }, "required": [ @@ -7497,39 +7508,19 @@ "filesystem_complete" ] }, - "Data_SnapshotRestoreReceipt": { - "type": "object", - "properties": { - "data": { - "$ref": "#/components/schemas/SnapshotRestoreReceipt" - } - }, - "required": [ - "data" - ] - }, - "SnapshotRestoreReceipt": { + "SnapshotRecoveryCheckout": { "type": "object", "additionalProperties": false, "properties": { - "origin": { - "$ref": "#/components/schemas/SnapshotOrigin" - }, - "intent_digest": { - "type": "string" - }, - "session_before": { - "type": "integer", - "format": "int64" - }, - "session_after": { + "version": { "type": "integer", - "format": "int64" + "format": "uint32", + "minimum": 0 }, - "events_digest": { - "type": "string" + "resolution_origin": { + "$ref": "#/components/schemas/SnapshotOrigin" }, - "checkout": { + "original": { "anyOf": [ { "$ref": "#/components/schemas/SnapshotRestoreCheckout" @@ -7538,17 +7529,17 @@ "type": "null" } ], - "description": "Missing on historical receipts; absence never authorizes checkout acknowledgement." + "description": "None preserves missing legacy proof, rather than claiming an empty set." + }, + "resolution": { + "$ref": "#/components/schemas/SnapshotRestoreCheckout" } }, "required": [ - "origin", - "intent_digest", - "session_before", - "session_after", - "events_digest" - ], - "description": "Exact native intent and Session batch consumed by an ordinary restore.\nThis is durable acknowledgement evidence, never authority to replay files." + "version", + "resolution_origin", + "resolution" + ] }, "SnapshotRestoreCheckout": { "type": "object", @@ -7642,6 +7633,59 @@ "directory" ] }, + "Data_SnapshotRestoreReceipt": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/SnapshotRestoreReceipt" + } + }, + "required": [ + "data" + ] + }, + "SnapshotRestoreReceipt": { + "type": "object", + "additionalProperties": false, + "properties": { + "origin": { + "$ref": "#/components/schemas/SnapshotOrigin" + }, + "intent_digest": { + "type": "string" + }, + "session_before": { + "type": "integer", + "format": "int64" + }, + "session_after": { + "type": "integer", + "format": "int64" + }, + "events_digest": { + "type": "string" + }, + "checkout": { + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotRestoreCheckout" + }, + { + "type": "null" + } + ], + "description": "Missing on historical receipts; absence never authorizes checkout acknowledgement." + } + }, + "required": [ + "origin", + "intent_digest", + "session_before", + "session_after", + "events_digest" + ], + "description": "Exact native intent and Session batch consumed by an ordinary restore.\nThis is durable acknowledgement evidence, never authority to replay files." + }, "Data_Array_of_SnapshotPreparation": { "type": "object", "properties": { diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index f862df659..ea2696e50 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -4792,6 +4792,14 @@ export type SnapshotRecovery = { export type SnapshotRecoveryAction = "keep" | "rollback"; +export type SnapshotRecoveryCheckout = { + version: number; + resolution_origin: SnapshotOrigin; + /** None preserves missing legacy proof, rather than claiming an empty set. */ + original?: SnapshotRestoreCheckout | null; + resolution: SnapshotRestoreCheckout; +}; + export type SnapshotRecoveryConfirmation = { journal_id: string; action: SnapshotRecoveryAction; @@ -4838,6 +4846,8 @@ export type SnapshotRecoveryReceipt = { * Missing on older receipts; those cannot authorize new journal writes. */ resolution_digest?: string | null; + /** Runtime-attested original and resolution resources; absent on legacy receipts. */ + checkout?: SnapshotRecoveryCheckout | null; }; export type SnapshotRecoveryResult = { From 4d80131218112c2619dc689368ccd4a58e2e739a Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 05:37:38 -0400 Subject: [PATCH 33/87] Preserve Git object formats and retained copy history --- crates/cyber-snapshot/src/copy/git/tests.rs | 262 ++++++++++++++++-- crates/cyber-snapshot/src/git.rs | 33 ++- crates/cyber-snapshot/src/lib.rs | 94 ++++++- .../implementation.md | 86 ++++++ .../managed-recovery.md | 24 ++ .../specs/snapshots-checkpoints/spec.md | 14 + .../complete-daily-coding-surfaces/tasks.md | 12 + 7 files changed, 497 insertions(+), 28 deletions(-) diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index db239abbc..b54a691b0 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -12,6 +12,9 @@ struct Repo { } impl Repo { fn new() -> Self { + Self::with_format("sha1") + } + fn with_format(format: &str) -> Self { let temp = tempfile::tempdir().unwrap(); let base = temp.path().canonicalize().unwrap(); let root = base.join("work"); @@ -19,7 +22,7 @@ impl Repo { std::fs::create_dir(&root).unwrap(); assert!( std::process::Command::new("git") - .args(["init", "--quiet"]) + .args(["init", "--quiet", &format!("--object-format={format}")]) .arg(&root) .status() .unwrap() @@ -1153,21 +1156,8 @@ async fn native_git_rejects_later_normalization_equivalent_owner_bytes() { } #[tokio::test] -async fn native_git_raw_capture_supports_sha256_shadow_graphs_and_encoded_bytes() { - let repo = Repo::new(); - // Test the graph codec at both object widths. This does not claim public - // discovery of SHA-256 user repositories (libgit2 discovery still refuses it). - let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); - std::fs::create_dir_all(&shadow.git_dir).unwrap(); - assert!( - std::process::Command::new("git") - .arg("--git-dir") - .arg(&shadow.git_dir) - .args(["init", "--bare", "--quiet", "--object-format=sha256"]) - .status() - .unwrap() - .success() - ); +async fn native_git_raw_capture_supports_sha256_user_repositories_and_encoded_bytes() { + let repo = Repo::with_format("sha256"); std::fs::write( repo.root.join(".gitattributes"), b"*.txt working-tree-encoding=UTF-16LE\n", @@ -1182,8 +1172,24 @@ async fn native_git_raw_capture_supports_sha256_shadow_graphs_and_encoded_bytes( .flat_map(u16::to_le_bytes) .collect(); std::fs::write(repo.root.join("file.txt"), &before).unwrap(); + assert!( + std::process::Command::new("git") + .arg("-C") + .arg(&repo.root) + .args(["add", "--all"]) + .status() + .unwrap() + .success() + ); + let user_index = std::fs::read(repo.root.join(".git/index")).unwrap(); + let user_config = std::fs::read(repo.root.join(".git/config")).unwrap(); let target = repo.track(&repo.root).await; assert_eq!(target.len(), 64); + assert!( + !repo + .snapshots + .owned_restore_available(repo.root.to_str().unwrap()) + ); std::fs::write(repo.root.join("file.txt"), &after).unwrap(); let recorded = repo.track(&repo.root).await; let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); @@ -1201,6 +1207,230 @@ async fn native_git_raw_capture_supports_sha256_shadow_graphs_and_encoded_bytes( ["file.txt"] ); assert_eq!(std::fs::read(repo.root.join("file.txt")).unwrap(), before); + assert_eq!( + std::fs::read(repo.root.join(".git/index")).unwrap(), + user_index + ); + assert_eq!( + std::fs::read(repo.root.join(".git/config")).unwrap(), + user_config + ); +} + +#[tokio::test] +async fn git_markers_refuse_non_git_admission_without_creating_identity_or_snapshot_state() { + let repo = Repo::with_format("sha256"); + let nested = repo.root.join("nested/deep"); + std::fs::create_dir_all(&nested).unwrap(); + for path in [&repo.root, &nested] { + assert!(!repo.snapshots.non_git(path.to_str().unwrap())); + assert!( + !repo + .snapshots + .owned_restore_available(path.to_str().unwrap()) + ); + } + assert!(!repo.root.join(".git/cyber-project-id").exists()); + assert!(!repo.data.exists()); + let missing = format!("copy:{}", "0".repeat(64)); + assert!( + repo.snapshots + .changed(repo.root.to_str().unwrap(), &missing, &missing) + .await + .is_err() + ); + assert!( + repo.snapshots + .diff(repo.root.to_str().unwrap(), &missing, &missing) + .await + .is_err() + ); + assert!(!repo.data.exists()); + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join(".git"), "malformed gitfile\n").unwrap(); + assert!(!repo.snapshots.non_git(root.to_str().unwrap())); + assert!( + repo.snapshots + .track(root.to_str().unwrap()) + .await + .unwrap_err() + .contains("fallback is disabled") + ); + std::fs::remove_file(root.join(".git")).unwrap(); + std::os::unix::fs::symlink("missing", root.join(".git")).unwrap(); + assert!(!repo.snapshots.non_git(root.to_str().unwrap())); + std::fs::remove_file(root.join(".git")).unwrap(); + assert!(repo.snapshots.non_git(root.to_str().unwrap())); + assert!(!repo.data.exists()); +} + +#[tokio::test] +async fn mismatched_git_shadow_format_refuses_capture_without_touching_source_or_index() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), "source").unwrap(); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + std::fs::create_dir_all(&shadow.git_dir).unwrap(); + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args(["init", "--bare", "--quiet", "--object-format=sha256"]) + .status() + .unwrap() + .success() + ); + let config = std::fs::read(shadow.git_dir.join("config")).unwrap(); + let error = repo + .snapshots + .track(repo.root.to_str().unwrap()) + .await + .unwrap_err(); + assert!(error.contains("object format differs"), "{error}"); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"source"); + assert_eq!( + std::fs::read(shadow.git_dir.join("config")).unwrap(), + config + ); + assert!(!shadow.git_dir.join("index").exists()); + assert!(!repo.root.join(".git/index").exists()); + assert!(!repo.data.join("snapshot/global").exists()); +} + +#[tokio::test] +async fn sha256_linked_worktree_uses_its_common_repository_format_and_scoped_git_images() { + let repo = Repo::with_format("sha256"); + assert!( + std::process::Command::new("git") + .arg("-C") + .arg(&repo.root) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "commit", + "--quiet", + "--allow-empty", + "-m", + "root" + ]) + .status() + .unwrap() + .success() + ); + let linked = repo._temp.path().canonicalize().unwrap().join("linked"); + assert!( + std::process::Command::new("git") + .arg("-C") + .arg(&repo.root) + .args(["worktree", "add", "--quiet", "--detach"]) + .arg(&linked) + .arg("HEAD") + .status() + .unwrap() + .success() + ); + let marker = std::fs::read(linked.join(".git")).unwrap(); + let location = linked.join("nested"); + std::fs::create_dir(&location).unwrap(); + std::fs::write(location.join("file"), "before").unwrap(); + let target = repo.track(&location).await; + assert_eq!(target.len(), 64); + std::fs::write(location.join("file"), "after").unwrap(); + let recorded = repo.track(&location).await; + let prepared = repo.prepare(&location, &target, &recorded, &recorded).await; + assert_eq!(prepared.prefix, "nested"); + assert_eq!(prepared.target.images["file"].bytes, b"before"); + repo.store(&location) + .restore_git_worker(prepared, None) + .unwrap(); + assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); + std::fs::write(location.join("file"), "after").unwrap(); + let paths = repo + .snapshots + .restore_reviewed( + location.to_str().unwrap(), + &target, + &recorded, + &recorded, + Arc::new(|| Ok(())), + ) + .await + .unwrap(); + assert_eq!(paths, ["nested/file"]); + assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); + assert_eq!(std::fs::read(linked.join(".git")).unwrap(), marker); + assert!( + !repo + .snapshots + .owned_restore_available(location.to_str().unwrap()) + ); +} + +#[tokio::test] +async fn retained_copy_history_stays_readable_after_sha256_repository_routing_is_corrected() { + let repo = Repo::with_format("sha256"); + let directory = repo.root.to_str().unwrap(); + std::fs::write(repo.root.join("file"), "before\n").unwrap(); + std::fs::write(repo.root.join("huge"), vec![b'x'; 65537]).unwrap(); + // Reproduce checkpoints captured by the prior erroneous fallback routing. + let store = repo.store(&repo.root); + let target = store.track().unwrap().tree; + std::fs::write(repo.root.join("file"), "after\n").unwrap(); + let recorded = store.track().unwrap().tree; + let excluded = store + .exclusions(&[target.clone(), recorded.clone()]) + .unwrap(); + assert!(!excluded.is_empty()); + assert!( + repo.snapshots + .changed(directory, &target, &recorded) + .await + .unwrap_err() + .contains("storage is busy") + ); + drop(store); + let git = repo.track(&repo.root).await; + assert_eq!(git.len(), 64); + assert_eq!( + repo.snapshots + .changed(directory, &target, &recorded) + .await + .unwrap(), + ["file"] + ); + assert_eq!( + repo.snapshots + .diff(directory, &target, &recorded) + .await + .unwrap() + .len(), + 1 + ); + let observed = repo + .snapshots + .exclusions(directory, vec![target.clone(), recorded, git.clone()]) + .await + .unwrap(); + for path in excluded { + assert!(observed.contains(&path)); + } + assert!( + repo.snapshots + .changed(directory, &target, &git) + .await + .unwrap_err() + .contains("different backends") + ); + assert!( + repo.snapshots + .diff(directory, &git, &target) + .await + .unwrap_err() + .contains("different backends") + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after\n"); } #[tokio::test] diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 2b9dc88e1..870770b44 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -80,6 +80,18 @@ impl Drop for PreviewIndex { } } +/// A marker only prevents fallback admission. It grants no Git/source authority +/// and creates no project identity cache when library discovery is unsupported. +pub(crate) fn repository_marker(location: &Path) -> bool { + let Ok(location) = std::fs::canonicalize(location) else { + return true; + }; + location.ancestors().any(|parent| { + !matches!(std::fs::symlink_metadata(parent.join(".git")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound) + }) +} + impl Shadow { pub fn locate(root: &Path, location: &Path) -> Option { let project = cyber_core::project::identify(location); @@ -151,11 +163,18 @@ impl Shadow { } async fn ensure_init(&self) -> Result<(), String> { + let mut source = self.clone(); + source.git_dir = self.common_dir.clone(); + let format = source.object_format().await?; if self.git_dir.join("HEAD").exists() { + if self.object_format().await? != format { + return Err("Git snapshot shadow object format differs from its repository".into()); + } return Ok(()); } std::fs::create_dir_all(&self.git_dir).map_err(|e| e.to_string())?; - self.run(&["init", "--quiet"]).await?; + self.run(&["init", "--quiet", &format!("--object-format={format}")]) + .await?; for (key, value) in [ ("core.autocrlf", "false"), ("core.fsmonitor", "false"), @@ -168,6 +187,18 @@ impl Shadow { std::fs::write(alternates, format!("{}\n", objects.display())).map_err(|e| e.to_string()) } + async fn object_format(&self) -> Result<&'static str, String> { + match self + .run_bounded(&["rev-parse", "--show-object-format=storage"], 16) + .await? + .as_slice() + { + b"sha1\n" => Ok("sha1"), + b"sha256\n" => Ok("sha256"), + _ => Err("Unsupported Git snapshot repository object format".into()), + } + } + pub async fn track(&self, ignore: &[String], max_bytes: u64) -> Result { self.ensure_init().await?; let skipped = self.large_untracked(max_bytes).await; diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 83542ff5d..75c7011e2 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -29,6 +29,16 @@ pub type ReferencesFn = dyn Fn() -> Result, String> + Send + Sy const DEFAULT_MAX_FILE_BYTES: u64 = 2 * 1024 * 1024; +fn copy_comparison(from: &str, to: &str) -> Result { + match (from.starts_with("copy:"), to.starts_with("copy:")) { + (true, true) => Ok(true), + (false, false) => Ok(false), + _ => { + Err("Snapshots use different backends; cross-backend comparison is unavailable".into()) + } + } +} + pub struct GitSnapshots { data_dir: PathBuf, config: Arc, @@ -92,6 +102,7 @@ impl GitSnapshots { .any(|root| Path::new(directory).starts_with(root)) && (self.config)(Path::new(directory)).get("snapshots") != Some(&Value::Bool(false)) && cyber_core::project::worktree_root(Path::new(directory)).is_none() + && !git::repository_marker(Path::new(directory)) } #[cfg(any(target_os = "macos", target_os = "linux"))] @@ -132,6 +143,60 @@ impl GitSnapshots { .map_err(|e| format!("Snapshot worker did not acknowledge completion: {e}"))? } + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn read_file_copy( + &self, + directory: String, + operation: impl FnOnce(copy::Store) -> Result + Send + 'static, + ) -> Result { + let options = self.copy_options(&directory); + let data = self.data_dir.clone(); + let lock = cyber_server::workspace::mutation_lock(Path::new(&directory)) + .map_err(|error| error.to_string())?; + let guard = lock.lock_owned().await; + tokio::task::spawn_blocking(move || { + let _guard = guard; + // Keep the existing native namespace lock through immutable reads; + // opening for inspection alone would lose the former reader fence. + let store = copy::Store::recover_existing(&data, &directory, options)?; + operation(store) + }) + .await + .map_err(|error| error.to_string())? + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn retained_exclusions( + &self, + directory: String, + trees: Vec, + ) -> Result, String> { + if trees.is_empty() { + return Ok(vec![]); + } + if self.non_git(&directory) { + return self + .read_file_copy(directory, move |store| store.exclusions(&trees)) + .await; + } + let (copies, git): (Vec<_>, Vec<_>) = + trees.into_iter().partition(|id| id.starts_with("copy:")); + let mut exclusions = BTreeSet::new(); + if !copies.is_empty() { + exclusions.extend( + self.read_file_copy(directory.clone(), move |store| store.exclusions(&copies)) + .await?, + ); + } + if !git.is_empty() { + let shadow = self + .shadow(&directory) + .ok_or("Git snapshot exclusion history is unavailable")?; + exclusions.extend(shadow.historical_exclusions(&git).await?); + } + Ok(exclusions.into_iter().collect()) + } + /// Inspect retained journals without restoring files or acknowledging native effects. pub async fn recovery(&self, directory: &str) -> Result, String> { #[cfg(any(target_os = "macos", target_os = "linux"))] @@ -602,6 +667,12 @@ impl Snapshots for GitSnapshots { } } let Some(shadow) = self.shadow(&directory) else { + if (self.config)(Path::new(&directory)).get("snapshots") + != Some(&Value::Bool(false)) + && git::repository_marker(Path::new(&directory)) + { + return Err("Git snapshots are unavailable for this repository; non-Git fallback is disabled".into()); + } return Ok(None); }; let (ignore, max) = self.options(Path::new(&directory)); @@ -620,10 +691,10 @@ impl Snapshots for GitSnapshots { ) -> BoxFuture<'_, Result, String>> { let (directory, from, to) = (directory.to_string(), from.to_string(), to.to_string()); Box::pin(async move { - if self.non_git(&directory) { + if copy_comparison(&from, &to)? || self.non_git(&directory) { #[cfg(any(target_os = "macos", target_os = "linux"))] return self - .file_copy(directory, move |store| store.changed(&from, &to)) + .read_file_copy(directory, move |store| store.changed(&from, &to)) .await; #[cfg(not(any(target_os = "macos", target_os = "linux")))] return Err("Non-Git snapshots are unavailable on this host".into()); @@ -647,10 +718,10 @@ impl Snapshots for GitSnapshots { ) -> BoxFuture<'_, Result, String>> { let (directory, from, to) = (directory.to_string(), from.to_string(), to.to_string()); Box::pin(async move { - if self.non_git(&directory) { + if copy_comparison(&from, &to)? || self.non_git(&directory) { #[cfg(any(target_os = "macos", target_os = "linux"))] return self - .file_copy(directory, move |store| store.diff(&from, &to)) + .read_file_copy(directory, move |store| store.diff(&from, &to)) .await; #[cfg(not(any(target_os = "macos", target_os = "linux")))] return Err("Non-Git snapshots are unavailable on this host".into()); @@ -670,15 +741,16 @@ impl Snapshots for GitSnapshots { let directory = directory.to_owned(); Box::pin(async move { #[cfg(any(target_os = "macos", target_os = "linux"))] - if self.non_git(&directory) { - return self - .file_copy(directory, move |store| store.exclusions(&trees)) - .await; + { + self.retained_exclusions(directory, trees).await } - if let Some(shadow) = self.shadow(&directory) { - return shadow.historical_exclusions(&trees).await; + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + if let Some(shadow) = self.shadow(&directory) { + return shadow.historical_exclusions(&trees).await; + } + Ok(vec![]) } - Ok(vec![]) }) } fn reviewed_restore_available(&self) -> bool { diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 3a205a3b6..a52131be4 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2474,3 +2474,89 @@ integration/native and real CLI tests cover the added behavior; generated SDK checks and real-client tests pass as recorded above. This review does not approve the full recovery delivery increment or close the explicitly retained managed Git, partial-construction, reclamation, crash/platform/client and CI acceptance gaps. + + +## Git object-format routing and retained-history inspection (2026-11-07) + +A new regression against production code at 0a7cfad4 initialized an actual SHA-256 +user repository and captured through the public Snapshots adapter. It failed with +a 69-character copy identity rather than a 64-character Git identity: libgit2's +read-only worktree discovery refused this repository, so non_git incorrectly +selected the copy backend. The former test manually initialized a SHA-256 shadow +inside a SHA-1 user repository and therefore did not prove public discovery. +This is a comparison against the prior local implementation, not an assertion +about unchanged main. + +Read-only ancestor marker checks now prevent fallback admission for unsupported, +malformed, dangling or unreadable Git metadata. Canonicalizing the Location covers +relative paths and nested Locations; a marker does not supply repository, native +worker or mutation authority. Classification creates no identity cache or snapshot +namespace. Capture reports unavailable repository discovery explicitly instead of +silently returning a new copy checkpoint. + +Before shadow initialization, bounded read-only Git metadata queries determine the +exact common repository storage format. Only SHA-1/SHA-256 are accepted; existing +shadow formats must match before capture/index changes. New shadows explicitly +initialize with the matching format and borrow only matching-format object stores. +The storage selector is documented in +[Git's versioned rev-parse manual](https://git-scm.com/docs/git-rev-parse/2.31.0#Documentation/git-rev-parse.txt---show-object-formatstorageinputoutput). +No new native Git source admission or checkout/reclamation authority is enabled. + +Pre-fix SHA-256 Locations may retain copy checkpoints. Changed-path/diff inspection +now dispatches by typed pair identity, rejects mixed-backend comparisons explicitly, +and opens only an existing Location-bound store. Historical exclusions group copy +and Git IDs separately before combining reports. Missing stores refuse without +creating metadata. This prevents routing correction from hiding old read-only +history; cross-backend restore/migration is still unimplemented and must be resolved +before complete managed Git delivery. Original checkpoints/journals are retained, +not rewritten or relabeled. + +Regression coverage uses actual SHA-256 user repositories and a linked worktree, +including nested Location scoping, UTF-16 canonical/raw images, exact native import, +legacy public reviewed restore of supported plain images, unchanged user index and +config, original gitfile preservation, malformed/dangling marker refusal, +mismatched existing shadow refusal and existing-only history inspection. Managed +Git admission remains disabled in all fixtures. These tests do not prove the full +runtime-owned Git callback, crash/recovery/platform or migration contract. + + +The code-review skill was applied to routing, initialization, read ports, tests +and spec changes. Review found and corrected a native reader-lifetime regression: +inspection-only Store opening lacked the lock formerly held by file_copy reads. +The existing-only read helper now holds the original native namespace lock through +its callback. A real held-Store regression refuses inspection with storage-busy, +then succeeds after the actual owner drops. Missing-store reads create no state. +This retains existing fencing; complete reference/reader/lease-aware reclamation +still requires its separate implementation and acceptance. + +Local macOS arm64 verification: the full locked snapshot suite passed 141 cases +(101 unit, 18 file-copy integration, 22 Git integration), with 2 ignored subprocess +fixtures; no doc tests were present. The extended linked-worktree public-restore +case, missing-store/no-creation case and native reader-contention case also passed +focused final-source checks. All-target locked snapshot Clippy passed with Rust +warnings denied. Format/diff and generated SDK consistency checks passed. Pinned +strict OpenSpec passed all 117 items; spec lint reported 0 errors and 25 warnings +without unchanged-main attribution. Native C compilation reported sandbox_init/ +sandbox_free_error deprecation warnings without origin attribution. + +GitHub state was rechecked through the GitHub skill: #92 and #93 are open, no open +PRs were returned, and fetched origin/main remains db9e171e. There is no new main +composition to merge. No PR/push, source-bound CI, native Linux/Windows, TUI/App, +full workspace or separate SDK execution is claimed for this format-routing +continuation. Public API/schema shapes did not change. + +Review verdict: approve the local format-routing and inspection correction; +0 critical and 0 major findings remain unresolved. The native reader-lock finding +was fixed and regression-tested. Existing snapshot adapter/native Store boundaries +are preserved. Full managed Git integration, cross-backend restore/migration, +configuration/filter races, partial-constructor recovery, reference-aware cleanup +and full client/platform acceptance remain open; neither issue nor parent tasks +are complete. + + +Final-source locked real CLI branch/recovery acceptance passed all 17 cases, +including partial-replacement kills, keep/rollback, restart/retry, stage/clear, +disabled snapshots and ordinary no-op receipts. These cases preserve unmanaged +file-copy and legacy Git behavior; the new SHA-256 fixtures above separately cover +actual repository-format discovery and scoped materialization. No managed Git +runtime activation or full goal acceptance is inferred from these results. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index fdff43612..0a8b9900f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -626,3 +626,27 @@ resolution digest. Native journal verification remains responsible for its own immutable intent; runtime checkout evidence is additionally attested by the writer. Legacy receipts omit the section and gain no marker authority. Missing original version-2 proof cannot be silently downgraded to legacy evidence. + + +### Git repository format routing + +Read-only presence of an ancestor .git marker SHALL prevent an unsupported or +unreadable Git repository from being admitted as an ordinary non-Git Location. +This guard SHALL NOT parse untrusted markers into mutation authority or create +project identity/cache files. Shadow initialization SHALL query the exact user's +common repository object format with bounded read-only Git metadata output, +initialize a matching SHA-1 or SHA-256 private repository, and refuse an existing +shadow with a different format before capture/index mutation. Public capture of a +SHA-256 user repository SHALL retain its Git identities rather than silently +returning copy identities. Source admission, native acknowledgement and reclamation +retain their independent gates. Unsupported formats SHALL fail explicitly rather +than selecting a different backend. + + +Retained historical copy checkpoints SHALL remain readable by their typed IDs for +changed-path, diff and exclusion inspection after Git discovery is corrected. +These reads SHALL require an existing Location-bound store and SHALL create no +replacement namespace. Comparing a copy checkpoint to a Git checkpoint SHALL +refuse explicitly rather than return an empty difference or reinterpret IDs. +This read-only compatibility grants no cross-backend restore/migration authority; +that acceptance remains required before complete managed Git delivery. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 28f4e729a..7ba78660a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -317,3 +317,17 @@ #### Scenario: Non-git project - **WHEN** a Session runs in an admitted folder without git - **THEN** edits remain rewindable through private Location-bound file-copy snapshots + + +#### Scenario: Git repository object format does not select a non-Git backend +- **WHEN** a Location has an ancestor Git marker but the library cannot discover its repository format +- **THEN** the snapshot adapter SHALL NOT classify it as an ordinary non-Git Location or enable non-Git owned restore +- **AND** Git capture SHALL initialize a private shadow with the exact user's supported SHA-1 or SHA-256 object format and refuse a mismatched existing shadow before capture/index mutation +- **AND** read-only backend classification SHALL create no identity cache or snapshot namespace, and malformed/unreadable repository metadata SHALL NOT grant fallback mutation authority + + +#### Scenario: Historical copy checkpoints remain inspectable after Git routing changes +- **GIVEN** a Git Location retains copy checkpoints produced by earlier fallback routing +- **WHEN** the caller inspects changed paths, diffs or historical exclusions +- **THEN** typed copy IDs SHALL select the existing Location-bound copy store without creating a missing namespace, and exclusions from copy and Git history SHALL remain reportable +- **AND** comparisons between copy and Git IDs SHALL refuse explicitly without source effects or fabricated empty differences; read compatibility SHALL NOT authorize cross-backend restore or migration diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 62f057baa..47d2bf0d8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -405,3 +405,15 @@ substituting a fresh Session lease or replaying source work. Legacy receipts gai no new claim authority. This continuation does not close M2–M8 or parent tasks: partial acquisition, actual managed Git source integration, reference-aware reclamation and full native/platform/client acceptance remain required. + + +Git format routing continuation: public capture now declines non-Git fallback when +an ancestor .git marker exists even if libgit2 cannot discover the repository. +Private shadows initialize with the user's exact supported object format and +mismatched existing shadows refuse before index mutation. Actual SHA-256 repository, +encoded-image and linked-worktree/subdirectory coverage replaces the former +manually initialized shadow-only graph fixture. Retained copy history stays readable +by typed IDs through existing namespaces; mixed-backend comparison refuses. This +does not admit managed Git source work or implement cross-backend restore/migration. +These compatibility/acceptance requirements, configuration races, partial +construction and reference-aware cleanup remain open with M2–M8 and parent tasks. From f2e8d7f287b33943b132df7a443860fd69d1cf9c Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 06:28:29 -0400 Subject: [PATCH 34/87] Bind native Git policy to configuration origins --- .../cyber-snapshot/src/copy/config_inputs.rs | 430 +++++++++++++++ crates/cyber-snapshot/src/copy/git/tests.rs | 89 ++++ crates/cyber-snapshot/src/copy/git_rules.rs | 7 +- crates/cyber-snapshot/src/copy/mod.rs | 1 + crates/cyber-snapshot/src/git.rs | 4 +- crates/cyber-snapshot/src/git/catalog.rs | 66 ++- crates/cyber-snapshot/src/git/config.rs | 490 ++++++++++++++++++ crates/cyber-snapshot/src/git/prepared.rs | 58 +-- .../implementation.md | 92 ++++ .../managed-recovery.md | 48 ++ .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 15 + 12 files changed, 1264 insertions(+), 43 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/config_inputs.rs create mode 100644 crates/cyber-snapshot/src/git/config.rs diff --git a/crates/cyber-snapshot/src/copy/config_inputs.rs b/crates/cyber-snapshot/src/copy/config_inputs.rs new file mode 100644 index 000000000..21daf94bb --- /dev/null +++ b/crates/cyber-snapshot/src/copy/config_inputs.rs @@ -0,0 +1,430 @@ +//! Read-only configuration observations; never source or acknowledgement authority. +use super::{ownership, storage}; +use std::collections::BTreeMap; +use std::io::Read; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; + +const FILE_LIMIT: u64 = 1024 * 1024; +const TOTAL_LIMIT: usize = 8 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub(crate) struct ConfigInputs { + environment: String, + files: BTreeMap>, + contexts: BTreeMap, + repository: Option, +} +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +struct DirectoryState { + canonical: PathBuf, + identity: String, +} +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +struct FileState { + canonical: PathBuf, + parent_identity: String, + generation: String, + hash: String, +} + +impl ConfigInputs { + pub(crate) fn capture(paths: Vec, environment: String) -> storage::Result { + if environment_digest()? != environment { + return Err("Git configuration environment changed".into()); + } + let mut files = BTreeMap::new(); + let mut bytes = 0; + for path in paths { + if !path.is_absolute() || path.as_os_str().len() > 8192 || files.len() >= 1024 { + return Err("Git configuration input paths exceed limits".into()); + } + if files.contains_key(&path) { + continue; + } + let (state, contents) = read(&path)?; + bytes += contents.len() + path.as_os_str().len(); + if bytes > TOTAL_LIMIT { + return Err("Git configuration inputs exceed limit".into()); + } + files.insert(path, state); + } + if environment_digest()? != environment { + return Err("Git configuration environment changed".into()); + } + Ok(Self { + environment, + files, + contexts: BTreeMap::new(), + repository: None, + }) + } + + pub(crate) fn bind_repository(&mut self, path: PathBuf) -> storage::Result<()> { + private_repository(&path)?; + self.repository = Some(path); + Ok(()) + } + + pub(crate) fn bind_contexts(&mut self, paths: Vec) -> storage::Result<()> { + if paths.len() > 4 { + return Err("Git configuration directory context exceeds limit".into()); + } + for path in paths { + let canonical = path.canonicalize().map_err(storage::fail)?; + let directory = cap_std::fs::Dir::open_ambient_dir(&path, cap_std::ambient_authority()) + .map_err(storage::fail)?; + let identity = storage::dir_identity(&directory)?; + let fresh = cap_std::fs::Dir::open_ambient_dir(&path, cap_std::ambient_authority()) + .map_err(storage::fail)?; + if path.canonicalize().map_err(storage::fail)? != canonical + || storage::dir_identity(&fresh)? != identity + { + return Err("Git configuration directory context changed".into()); + } + self.contexts.insert( + path, + DirectoryState { + canonical, + identity, + }, + ); + } + Ok(()) + } + + pub(crate) fn retains(&self, roots: &Self) -> bool { + self.environment == roots.environment + && self.repository == roots.repository + && self.contexts == roots.contexts + && roots + .files + .iter() + .all(|(path, state)| self.files.get(path) == Some(state)) + } + + pub(crate) fn verify(&self) -> storage::Result<()> { + let mut current = Self::capture( + self.files.keys().cloned().collect(), + self.environment.clone(), + )?; + current.bind_contexts(self.contexts.keys().cloned().collect())?; + if let Some(repository) = &self.repository { + current.bind_repository(repository.clone())?; + } + if current != *self { + return Err("Git configuration inputs changed".into()); + } + Ok(()) + } +} + +fn open(path: &Path) -> std::io::Result { + // Git configuration follows file links. This grants only read-only policy + // proof; native source image reads keep their separate no-follow constraints. + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(path) +} + +fn read(path: &Path) -> storage::Result<(Option, Vec)> { + let mut file = match open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok((None, vec![])), + Err(_) => return Err("Cannot read Git configuration input".into()), + }; + let metadata = file.metadata().map_err(storage::fail)?; + if !metadata.is_file() || metadata.len() > FILE_LIMIT { + return Err("Unsupported or over-limit Git configuration input".into()); + } + let generation = ownership::generation(&file)?; + let canonical = path.canonicalize().map_err(storage::fail)?; + let parent = cap_std::fs::Dir::open_ambient_dir( + canonical.parent().ok_or("Missing config parent")?, + cap_std::ambient_authority(), + ) + .map_err(storage::fail)?; + let parent_identity = storage::dir_identity(&parent)?; + let mut bytes = Vec::new(); + Read::by_ref(&mut file) + .take(FILE_LIMIT + 1) + .read_to_end(&mut bytes) + .map_err(storage::fail)?; + if bytes.len() as u64 != metadata.len() + || ownership::generation(&file)? != generation + || ownership::generation(&open(path).map_err(storage::fail)?)? != generation + || path.canonicalize().map_err(storage::fail)? != canonical + || storage::dir_identity( + &cap_std::fs::Dir::open_ambient_dir( + canonical.parent().ok_or("Missing config parent")?, + cap_std::ambient_authority(), + ) + .map_err(storage::fail)?, + )? != parent_identity + { + return Err("Git configuration changed while reading".into()); + } + Ok(( + Some(FileState { + canonical, + parent_identity, + generation, + hash: storage::digest(&bytes), + }), + bytes, + )) +} + +fn private_repository(path: &Path) -> storage::Result<()> { + for name in ["commondir", "gitdir", "reftable"] { + match std::fs::symlink_metadata(path.join(name)) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + _ => return Err("Unsupported private Git reference layout".into()), + } + } + let head = path.join("HEAD"); + if std::fs::symlink_metadata(&head) + .map_err(storage::fail)? + .file_type() + .is_symlink() + { + return Err("Linked private Git HEAD lacks native policy proof".into()); + } + let (_, bytes) = read(&head)?; + let head = std::str::from_utf8(&bytes).map_err(|_| "Invalid private Git HEAD")?; + let target = head + .trim_end_matches('\n') + .strip_prefix("ref: ") + .ok_or("Unsupported private Git HEAD")?; + if !target.starts_with("refs/heads/") || !reference_name(target) { + return Err("Unsupported private Git HEAD reference".into()); + } + if std::fs::read_dir(path.join("refs/heads")) + .map_err(storage::fail)? + .next() + .is_some() + { + return Err("Private Git branch references lack native policy proof".into()); + } + let (_, packed) = read(&path.join("packed-refs"))?; + packed_references(&packed) +} + +fn packed_references(packed: &[u8]) -> storage::Result<()> { + for line in packed + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty() && !line.starts_with(b"#")) + { + let line = std::str::from_utf8(line).map_err(|_| "Invalid packed Git references")?; + if let Some(peeled) = line.strip_prefix('^') { + if !object_id(peeled) { + return Err("Invalid peeled Git reference".into()); + } + continue; + } + let (object, name) = line.split_once(' ').ok_or("Invalid packed Git reference")?; + if !object_id(object) || !reference_name(name) || name.starts_with("refs/heads/") { + return Err("Unsupported packed private Git reference".into()); + } + } + Ok(()) +} + +fn object_id(value: &str) -> bool { + matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} +fn reference_name(value: &str) -> bool { + value.starts_with("refs/") + && !["..", "@{", "//"].iter().any(|part| value.contains(part)) + && !value + .bytes() + .any(|byte| byte <= b' ' || byte == 127 || b"~^:?*[\\".contains(&byte)) + && value.split('/').all(|part| { + !part.is_empty() + && !part.starts_with('.') + && !part.ends_with('.') + && !part.ends_with(".lock") + }) +} + +pub(crate) fn environment_digest() -> storage::Result { + let mut values = BTreeMap::new(); + let mut bytes = 0; + for (key, value) in std::env::vars_os() { + let key = key.to_str().ok_or("Non-UTF-8 environment key")?; + if !(key.starts_with("GIT_") + || key.starts_with("LC_") + || matches!(key, "HOME" | "XDG_CONFIG_HOME" | "PATH" | "LANG")) + || crate::git::GIT_ENV.contains(&key) + { + continue; + } + let value = value + .to_str() + .ok_or("Non-UTF-8 Git configuration environment")?; + bytes += key.len() + value.len(); + if values.len() >= 1024 || bytes > 256 * 1024 || value.contains(['\n', '\r']) { + return Err("Unsupported or over-limit Git configuration environment".into()); + } + values.insert(key.to_owned(), value.to_owned()); + } + // Only a digest enters capture fingerprints; configuration secrets stay out + // of the handoff, checkpoint tags and Debug output. + Ok(storage::digest( + &serde_json::to_vec(&("git-config-environment-v1", values)).map_err(storage::fail)?, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_state_binds_missing_candidates_replacements_and_limits() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("config"); + let missing = + ConfigInputs::capture(vec![path.clone()], environment_digest().unwrap()).unwrap(); + missing.verify().unwrap(); + std::fs::write(&path, "same contents").unwrap(); + assert!(missing.verify().is_err()); + let observed = + ConfigInputs::capture(vec![path.clone()], environment_digest().unwrap()).unwrap(); + observed.verify().unwrap(); + let replacement = temp.path().join("replacement"); + std::fs::write(&replacement, "same contents").unwrap(); + std::fs::rename(replacement, &path).unwrap(); + assert!(observed.verify().is_err()); + std::fs::write(&path, vec![0; FILE_LIMIT as usize + 1]).unwrap(); + assert!(ConfigInputs::capture(vec![path], environment_digest().unwrap()).is_err()); + } + + #[test] + fn linked_config_remains_readable_but_same_byte_retargeting_is_stale() { + let temp = tempfile::tempdir().unwrap(); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + let route = temp.path().join("config"); + std::fs::write(&first, "same").unwrap(); + std::fs::write(&second, "same").unwrap(); + std::os::unix::fs::symlink(&first, &route).unwrap(); + let proof = + ConfigInputs::capture(vec![route.clone()], environment_digest().unwrap()).unwrap(); + proof.verify().unwrap(); + std::fs::remove_file(&route).unwrap(); + std::os::unix::fs::symlink(&second, route).unwrap(); + assert!(proof.verify().is_err()); + } + + #[test] + fn a_hardlinked_config_alias_still_binds_its_canonical_parent() { + let temp = tempfile::tempdir().unwrap(); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + std::fs::create_dir(&first).unwrap(); + std::fs::create_dir(&second).unwrap(); + std::fs::write(first.join("config"), "same").unwrap(); + std::fs::hard_link(first.join("config"), second.join("config")).unwrap(); + let route = temp.path().join("route"); + std::os::unix::fs::symlink(first.join("config"), &route).unwrap(); + let proof = + ConfigInputs::capture(vec![route.clone()], environment_digest().unwrap()).unwrap(); + std::fs::remove_file(&route).unwrap(); + std::os::unix::fs::symlink(second.join("config"), route).unwrap(); + assert!(proof.verify().is_err()); + } + + #[test] + fn private_reference_layout_allows_snapshot_packing_but_refuses_new_branch_or_admin_authority() + { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(root.join("refs/heads")).unwrap(); + std::fs::write(root.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + let mut proof = ConfigInputs::capture(vec![], environment_digest().unwrap()).unwrap(); + proof.bind_repository(root.clone()).unwrap(); + proof.verify().unwrap(); + let object = "a".repeat(40); + std::fs::write( + root.join("packed-refs"), + format!("{object} refs/cyber/snapshots/{object}\n"), + ) + .unwrap(); + proof.verify().unwrap(); + std::fs::write(root.join("refs/heads/main"), format!("{object}\n")).unwrap(); + assert!(proof.verify().is_err()); + std::fs::remove_file(root.join("refs/heads/main")).unwrap(); + std::fs::write( + root.join("packed-refs"), + format!("{object} refs/heads/main\n"), + ) + .unwrap(); + assert!(proof.verify().is_err()); + std::fs::remove_file(root.join("packed-refs")).unwrap(); + std::fs::write(root.join("commondir"), "elsewhere\n").unwrap(); + assert!(proof.verify().is_err()); + std::fs::remove_file(root.join("commondir")).unwrap(); + std::fs::create_dir(root.join("reftable")).unwrap(); + assert!(proof.verify().is_err()); + } + + #[test] + fn directory_context_binds_the_physical_conditional_include_route() { + let temp = tempfile::tempdir().unwrap(); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + let route = temp.path().join("git-dir"); + std::fs::create_dir(&first).unwrap(); + std::fs::create_dir(&second).unwrap(); + std::os::unix::fs::symlink(&first, &route).unwrap(); + let mut proof = ConfigInputs::capture(vec![], environment_digest().unwrap()).unwrap(); + proof.bind_contexts(vec![route.clone()]).unwrap(); + proof.verify().unwrap(); + std::fs::write(first.join("unrelated-object"), "object").unwrap(); + proof.verify().unwrap(); + std::fs::remove_file(&route).unwrap(); + std::os::unix::fs::symlink(&second, route).unwrap(); + assert!(proof.verify().is_err()); + } + + #[test] + fn environment_is_bound_without_retaining_cleartext_values() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "copy::config_inputs::tests::environment_child", + ]) + .env( + "CYBER_CONFIG_PARENT_FINGERPRINT", + environment_digest().unwrap(), + ) + .env("GIT_CONFIG_COUNT", "1") + .env("GIT_CONFIG_KEY_0", "fixture.private") + .env("GIT_CONFIG_VALUE_0", "fixture-secret-value-unique") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + #[ignore = "subprocess fixture with isolated Git configuration environment"] + fn environment_child() { + let actual = environment_digest().unwrap(); + assert_ne!( + actual, + std::env::var("CYBER_CONFIG_PARENT_FINGERPRINT").unwrap() + ); + let proof = ConfigInputs::capture(vec![], actual).unwrap(); + let encoded = serde_json::to_string(&proof).unwrap(); + assert!(!encoded.contains("fixture-secret-value-unique")); + let mut changed = proof; + changed.environment = "0".repeat(64); + assert!(changed.verify().is_err()); + } +} diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index b54a691b0..0dda634f6 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -10,6 +10,95 @@ struct Repo { data: PathBuf, snapshots: Arc, } + +#[tokio::test] +async fn changed_git_config_locator_after_preparation_refuses_native_source_effects() { + let repo = Repo::new(); + repo.track(&repo.root).await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + let first = repo._temp.path().join("first-ignore"); + let second = repo._temp.path().join("second-ignore"); + std::fs::write(&first, "").unwrap(); + std::fs::write(&second, "").unwrap(); + let configure = |path: &Path| { + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args(["config", "core.excludesfile"]) + .arg(path) + .status() + .unwrap() + .success() + ) + }; + configure(&first); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + configure(&second); + let store = repo.store(&repo.root); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn native_git_config_proof_rejects_include_creation_changes_and_conditional_head_changes() { + for kind in ["missing", "included", "head"] { + let repo = Repo::new(); + repo.track(&repo.root).await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + let include = shadow.git_dir.join("native-policy.include"); + let key = if kind == "head" { + let head = std::fs::read_to_string(shadow.git_dir.join("HEAD")).unwrap(); + format!( + "includeIf.onbranch:{}.path", + head.trim().strip_prefix("ref: refs/heads/").unwrap() + ) + } else { + "include.path".into() + }; + if kind != "missing" { + std::fs::write(&include, "[core]\nignorecase=false\n").unwrap(); + } + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&shadow.git_dir) + .args(["config", &key, "native-policy.include"]) + .status() + .unwrap() + .success() + ); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + assert_eq!(store.preview_git(&prepared).unwrap().len(), 1); + if kind == "head" { + std::fs::write( + shadow.git_dir.join("HEAD"), + "ref: refs/heads/changed-private-branch\n", + ) + .unwrap(); + } else { + std::fs::write(&include, "[core]\nignorecase=true\n").unwrap(); + } + assert!(store.restore_git_worker(prepared, None).is_err(), "{kind}"); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} + impl Repo { fn new() -> Self { Self::with_format("sha1") diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs index f64101ad7..051920039 100644 --- a/crates/cyber-snapshot/src/copy/git_rules.rs +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -30,6 +30,7 @@ impl Rules { Self::open(&prepared.worktree, &prepared.prefix, &prepared.ignore) } fn open(worktree: &Path, prefix: &str, inputs: &IgnoreInputs) -> Result { + inputs.config.verify().map_err(|_| RestoreError::Stale)?; let mut rules = Self { root: Dir::open_ambient_dir(worktree, cap_std::ambient_authority()).map_err(failed)?, path: worktree.to_path_buf(), @@ -145,7 +146,7 @@ impl Rules { (left.source, &left.path, &left.hash).cmp(&(right.source, &right.path, &right.hash)) }); let bytes = serde_json::to_vec(&( - "git-policy-capture-v1", + "git-policy-capture-v2", &self.path, &self.inputs, bindings, @@ -155,6 +156,10 @@ impl Rules { Ok(blake3::hash(&bytes).to_hex().to_string()) } pub(super) fn verify(&self) -> Result<(), RestoreError> { + self.inputs + .config + .verify() + .map_err(|_| RestoreError::Stale)?; for binding in &self.bindings { let bytes = read(&self.root, &binding.path, binding.source)?; if bytes diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index ad6f806a8..4fb559527 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,4 +1,5 @@ //! Location-bound native recovery journals and private non-Git BLAKE3 snapshots. +pub(crate) mod config_inputs; mod git; pub(crate) mod git_capture; pub(crate) mod git_rules; diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 870770b44..d3c0cda7e 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -7,6 +7,8 @@ use std::sync::{Arc, Mutex}; use cyber_server::runtime::{FileDiff, Snapshot}; use sha1::{Digest, Sha1}; mod catalog; +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod config; #[path = "git_history.rs"] pub(crate) mod history; mod images; @@ -31,7 +33,7 @@ pub(crate) struct Shadow { } /// Environment variables that would redirect git away from the shadow repository. -const GIT_ENV: &[&str] = &[ +pub(crate) const GIT_ENV: &[&str] = &[ "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", diff --git a/crates/cyber-snapshot/src/git/catalog.rs b/crates/cyber-snapshot/src/git/catalog.rs index eecee4c5e..224427a1e 100644 --- a/crates/cyber-snapshot/src/git/catalog.rs +++ b/crates/cyber-snapshot/src/git/catalog.rs @@ -22,6 +22,16 @@ async fn bounded(reader: impl AsyncRead + Unpin, limit: usize) -> Result impl Shadow { pub(super) async fn run_bounded(&self, args: &[&str], limit: usize) -> Result, String> { + self.run_bounded_for(args, limit, std::time::Duration::from_secs(10)) + .await + } + + async fn run_bounded_for( + &self, + args: &[&str], + limit: usize, + timeout: std::time::Duration, + ) -> Result, String> { let mut child = self .command(args) .stdout(Stdio::piped()) @@ -33,16 +43,23 @@ impl Shadow { .stderr .take() .ok_or("Missing Git metadata diagnostics")?; - let reads = tokio::try_join!(bounded(stdout, limit), bounded(stderr, 64 * 1024)); - let (bytes, diagnostics) = match reads { - Ok(reads) => reads, + let completion = tokio::time::timeout(timeout, async { + let (bytes, diagnostics) = + tokio::try_join!(bounded(stdout, limit), bounded(stderr, 64 * 1024))?; + let status = child.wait().await.map_err(|error| error.to_string())?; + Ok((bytes, diagnostics, status)) + }) + .await + .map_err(|_| "Git snapshot metadata timed out".to_owned()) + .and_then(|result| result); + let (bytes, diagnostics, status) = match completion { + Ok(result) => result, Err(error) => { let _ = child.kill().await; let _ = child.wait().await; return Err(error); } }; - let status = child.wait().await.map_err(|error| error.to_string())?; if !status.success() { return Err(format!( "Git snapshot metadata: {}", @@ -197,4 +214,45 @@ mod tests { assert_eq!(bounded(&b"abc"[..], 3).await.unwrap(), b"abc"); assert!(bounded(&b"abcd"[..], 3).await.is_err()); } + + #[tokio::test] + #[cfg(unix)] + async fn stalled_configuration_metadata_is_killed_awaited_and_reader_remains_usable() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + assert!( + std::process::Command::new("git") + .args(["init", "--quiet"]) + .arg(&root) + .status() + .unwrap() + .success() + ); + let shadow = Shadow::locate(&root.join("data"), &root).unwrap(); + shadow.ensure_init().await.unwrap(); + let path = root.join("blocked.config"); + assert!( + std::process::Command::new("mkfifo") + .arg(&path) + .status() + .unwrap() + .success() + ); + let args = ["config", "--file", path.to_str().unwrap(), "--list"]; + assert!( + shadow + .run_bounded_for(&args, 1024, std::time::Duration::from_millis(50)) + .await + .unwrap_err() + .contains("timed out") + ); + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, "[fixture]\nvalue=ok\n").unwrap(); + let output = shadow.run_bounded(&args, 1024).await.unwrap(); + assert!( + String::from_utf8(output) + .unwrap() + .contains("fixture.value=ok") + ); + } } diff --git a/crates/cyber-snapshot/src/git/config.rs b/crates/cyber-snapshot/src/git/config.rs new file mode 100644 index 000000000..a97263495 --- /dev/null +++ b/crates/cyber-snapshot/src/git/config.rs @@ -0,0 +1,490 @@ +//! Bind config resolution to native read-only observations, not subprocess authority. +use super::Shadow; +use crate::copy::config_inputs::{ConfigInputs, environment_digest}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +#[derive(PartialEq, Eq)] +pub(super) struct Inventory { + values: Vec, + paths: BTreeSet, +} + +impl Inventory { + pub(super) fn selected(&self, key: &str) -> Result>, String> { + Ok(records(&self.values)? + .into_iter() + .rev() + .find(|(_, name, _)| *name == key) + .map(|(_, _, value)| value)) + } +} + +impl Shadow { + pub(super) async fn config_inputs(&self) -> Result<(ConfigInputs, Inventory), String> { + let environment = environment_digest()?; + let contexts = vec![self.worktree.clone(), self.worktree.join(&self.git_dir)]; + let roots = self.config_roots().await?; + let original = capture(roots.clone(), environment.clone(), contexts.clone()).await?; + let inventory = self.config_inventory().await?; + let paths = roots + .into_iter() + .chain(inventory.paths.iter().cloned()) + .collect(); + let proof = capture(paths, environment, contexts).await?; + if !proof.retains(&original) { + return Err("Git configuration roots changed during resolution".into()); + } + Ok((proof, inventory)) + } + + pub(super) async fn verify_config_inputs( + &self, + proof: &ConfigInputs, + inventory: &Inventory, + ) -> Result<(), String> { + if &self.config_inventory().await? != inventory { + return Err("Git configuration origins changed during resolution".into()); + } + let proof = proof.clone(); + tokio::task::spawn_blocking(move || proof.verify()) + .await + .map_err(|error| error.to_string())? + } + + async fn config_roots(&self) -> Result, String> { + let mut paths = BTreeSet::from([ + self.worktree.join(self.git_dir.join("config")), + self.worktree.join(self.git_dir.join("config.worktree")), + self.worktree.join(self.git_dir.join("HEAD")), + ]); + let no_system = std::env::var("GIT_CONFIG_NOSYSTEM").is_ok_and(|value| { + matches!( + value.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }); + for name in ["GIT_CONFIG_SYSTEM", "GIT_CONFIG_GLOBAL"] { + if (name == "GIT_CONFIG_SYSTEM" && no_system) + || std::env::var_os(name).is_some_and(|value| value.is_empty()) + { + continue; + } + let output = self.run_bounded(&["var", name], 32 * 1024).await?; + let text = std::str::from_utf8(&output).map_err(|_| "Non-UTF-8 Git config roots")?; + let text = text + .strip_suffix('\n') + .ok_or("Invalid Git configuration roots")?; + for path in text.split('\n') { + if path.is_empty() || path.contains('\0') { + return Err("Invalid Git configuration root path".into()); + } + paths.insert(self.worktree.join(path)); + } + } + if let Some(path) = std::env::var_os("GIT_CONFIG").filter(|path| !path.is_empty()) { + paths.insert(self.worktree.join(path)); + } + Ok(paths.into_iter().collect()) + } + + async fn config_inventory(&self) -> Result { + let values = self + .run_bounded( + &["config", "--null", "--includes", "--show-origin", "--list"], + 1024 * 1024, + ) + .await?; + let entries = records(&values)?; + for (_, key, _) in &entries { + if !supported_condition(key) { + return Err( + "Conditional Git configuration lacks complete native dependency proof".into(), + ); + } + } + let mut paths = BTreeSet::new(); + for (origin, _, _) in &entries { + if let Some(path) = file_origin(origin)? { + paths.insert(self.worktree.join(path)); + } + } + let includes: Vec<_> = entries + .iter() + .filter(|(_, key, _)| include_key(key)) + .collect(); + let raw = includes + .iter() + .map(|(_, _, value)| { + value + .filter(|value| !value.is_empty()) + .ok_or_else(|| "Invalid Git include target".to_owned()) + }) + .collect::, _>>()?; + let expanded = self.expand_config_paths(&raw).await?; + for ((origin, _, _), path) in includes.into_iter().zip(expanded) { + let parent = file_origin(origin)? + .and_then(Path::parent) + .unwrap_or(&self.worktree); + paths.insert(self.worktree.join(parent).join(path)); + } + if paths.len() > 1024 { + return Err("Git configuration origin count exceeds limit".into()); + } + Ok(Inventory { values, paths }) + } + + pub(super) async fn expand_config_paths(&self, raw: &[&str]) -> Result, String> { + if raw.is_empty() { + return Ok(vec![]); + } + if raw.len() > 1024 || raw.iter().map(|value| value.len()).sum::() > 64 * 1024 { + return Err("Git configuration path expansion exceeds limit".into()); + } + // Expand only values from the bound inventory. A second config lookup + // could select a different conditional include between observations. + let key = format!( + "cybernativepolicy.path{}", + ulid::Ulid::new().to_string().to_ascii_lowercase() + ); + let mut args = Vec::new(); + for value in raw { + if value.starts_with('~') && !value.starts_with("~/") { + return Err("Named-user Git home expansion lacks native policy proof".into()); + } + args.extend(["-c".to_owned(), format!("{key}={value}")]); + } + args.extend( + ["config", "--null", "--show-origin", "--path", "--get-all"].map(str::to_owned), + ); + args.push(key); + let output = self + .run_bounded( + &args.iter().map(String::as_str).collect::>(), + 1024 * 1024, + ) + .await?; + let values = pairs(&output)?; + if values.len() != raw.len() || values.iter().any(|(origin, _)| *origin != "command line:") + { + return Err("Git path expansion origin differs".into()); + } + Ok(values + .into_iter() + .map(|(_, path)| PathBuf::from(path)) + .collect()) + } + + pub(super) async fn config_boolean(&self, key: &str, raw: &str) -> Result { + let output = self + .run_bounded( + &[ + "-c", + &format!("{key}={raw}"), + "config", + "--bool", + "--get", + key, + ], + 16, + ) + .await?; + match output.as_slice() { + b"true\n" => Ok(true), + b"false\n" => Ok(false), + _ => Err("Invalid Git configuration boolean expansion".into()), + } + } +} + +async fn capture( + paths: Vec, + environment: String, + contexts: Vec, +) -> Result { + tokio::task::spawn_blocking(move || { + let mut proof = ConfigInputs::capture(paths, environment)?; + let repository = contexts + .get(1) + .ok_or("Missing private Git configuration context")? + .clone(); + proof.bind_contexts(contexts)?; + proof.bind_repository(repository)?; + Ok(proof) + }) + .await + .map_err(|error| error.to_string())? +} + +fn include_key(key: &str) -> bool { + key == "include.path" || (key.starts_with("includeif.") && key.ends_with(".path")) +} + +fn supported_condition(key: &str) -> bool { + let Some(condition) = key + .strip_prefix("includeif.") + .and_then(|value| value.strip_suffix(".path")) + else { + return true; + }; + if let Some(pattern) = condition + .strip_prefix("gitdir:") + .or_else(|| condition.strip_prefix("gitdir/i:")) + { + return !pattern.starts_with('~') || pattern.starts_with("~/"); + } + condition.starts_with("onbranch:") +} + +fn file_origin(origin: &str) -> Result, String> { + if let Some(path) = origin.strip_prefix("file:").filter(|path| !path.is_empty()) { + return Ok(Some(Path::new(path))); + } + if origin == "command line:" { + return Ok(None); + } + Err("Unsupported Git configuration origin".into()) +} + +type Record<'a> = (&'a str, &'a str, Option<&'a str>); +fn pairs(bytes: &[u8]) -> Result, String> { + if bytes.is_empty() { + return Ok(vec![]); + } + let bytes = bytes + .strip_suffix(&[0]) + .ok_or("Invalid Git config origin framing")?; + let parts: Vec<_> = bytes.split(|byte| *byte == 0).collect(); + if parts.len() % 2 != 0 || parts.len() > 32768 { + return Err("Invalid or over-limit Git configuration origins".into()); + } + parts + .as_chunks::<2>() + .0 + .iter() + .map(|pair| { + let origin = std::str::from_utf8(pair[0]).map_err(|_| "Non-UTF-8 Git config origin")?; + let entry = std::str::from_utf8(pair[1]).map_err(|_| "Non-UTF-8 Git config value")?; + if origin.is_empty() { + return Err("Empty Git configuration origin".into()); + } + Ok((origin, entry)) + }) + .collect() +} + +fn records(bytes: &[u8]) -> Result>, String> { + pairs(bytes)? + .into_iter() + .map(|(origin, entry)| { + let (key, value) = entry + .split_once('\n') + .map_or((entry, None), |(key, value)| (key, Some(value))); + if key.is_empty() { + return Err("Empty Git configuration key".into()); + } + Ok((origin, key, value)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn origin_records_keep_literal_paths_and_reject_incomplete_frames() { + let values = + b"file:/tmp/a\nconfig\0include.path\n../with space\0command line:\0core.bare\nfalse\0"; + let parsed = records(values).unwrap(); + assert_eq!( + parsed[0], + ("file:/tmp/a\nconfig", "include.path", Some("../with space")) + ); + assert_eq!(parsed[1], ("command line:", "core.bare", Some("false"))); + assert!(records(b"file:/tmp/config\0unpaired\0file:missing\0").is_err()); + assert!(records(b"file:/tmp/config\0key\nvalue").is_err()); + assert!(file_origin("blob:unknown").is_err()); + assert!(include_key("includeif.onbranch:main.path")); + assert!(!include_key("include.other.path")); + assert!(!supported_condition( + "includeif.hasconfig:remote.*.url:*.path" + )); + assert!(!supported_condition( + "includeif.gitdir:~named-user/project/.path" + )); + let inventory = Inventory { + values: b"file:a\0core.ignorecase\nfalse\0command line:\0core.ignorecase\0".to_vec(), + paths: BTreeSet::new(), + }; + assert_eq!(inventory.selected("core.ignorecase").unwrap(), Some(None)); + } + + #[tokio::test] + async fn expansion_uses_supplied_values_instead_of_reselecting_mutable_settings() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let work = root.join("work"); + std::fs::create_dir(&work).unwrap(); + assert!( + std::process::Command::new("git") + .args(["init", "--quiet"]) + .arg(&work) + .status() + .unwrap() + .success() + ); + let shadow = Shadow::locate(&root.join("data"), &work).unwrap(); + shadow.ensure_init().await.unwrap(); + shadow + .run(&["config", "core.ignorecase", "true"]) + .await + .unwrap(); + shadow + .run(&["config", "core.excludesfile", "unrelated-current-setting"]) + .await + .unwrap(); + let values = shadow + .expand_config_paths(&["../literal with space", ""]) + .await + .unwrap(); + assert_eq!( + values, + [PathBuf::from("../literal with space"), PathBuf::new()] + ); + assert!( + !shadow + .config_boolean("core.ignorecase", "false") + .await + .unwrap() + ); + assert!( + shadow + .expand_config_paths(&["~named-user/ignore"]) + .await + .is_err() + ); + shadow + .run(&[ + "config", + "includeIf.hasconfig:remote.*.url:example.path", + "missing-conditional-config", + ]) + .await + .unwrap(); + assert!( + shadow + .ignore_inputs() + .await + .unwrap_err() + .contains("dependency proof") + ); + std::fs::write(work.join("file"), "legacy-readable").unwrap(); + let snapshot = shadow.track(&[], 65536).await.unwrap(); + assert!( + snapshot + .skipped + .iter() + .any(|value| value.contains("native Git policy/raw-state proof unavailable")) + ); + assert!( + shadow + .snapshot_policy(&snapshot.tree) + .await + .unwrap() + .is_none() + ); + assert_eq!( + shadow + .blob(&snapshot.tree, "file") + .await + .unwrap() + .unwrap() + .1, + b"legacy-readable" + ); + } + + #[test] + fn missing_global_config_candidate_is_bound_in_an_isolated_process() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "git::config::tests::global_root_child", + ]) + .env("CYBER_CONFIG_ROOT_FIXTURE", &root) + .env("GIT_CONFIG_GLOBAL", root.join("global.config")) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_COUNT", "0") + .env_remove("GIT_CONFIG_PARAMETERS") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[tokio::test] + #[ignore = "subprocess fixture with isolated global configuration candidate"] + async fn global_root_child() { + use cyber_server::runtime::{SnapshotRestoreRequest, Snapshots}; + let root = PathBuf::from(std::env::var_os("CYBER_CONFIG_ROOT_FIXTURE").unwrap()); + let global = PathBuf::from(std::env::var_os("GIT_CONFIG_GLOBAL").unwrap()); + assert_eq!(global, root.join("global.config")); + assert!(!global.exists()); + let work = root.join("work"); + std::fs::create_dir(&work).unwrap(); + assert!( + std::process::Command::new("git") + .args(["init", "--quiet"]) + .arg(&work) + .status() + .unwrap() + .success() + ); + let data = root.join("data"); + let snapshots = + crate::GitSnapshots::new(data.clone(), std::sync::Arc::new(|_| serde_json::json!({}))); + let directory = work.to_str().unwrap(); + std::fs::write(work.join("file"), "before").unwrap(); + let target = snapshots.track(directory).await.unwrap().unwrap().tree; + std::fs::write(work.join("file"), "after").unwrap(); + let recorded = snapshots.track(directory).await.unwrap().unwrap().tree; + let prepared = snapshots + .shadow(directory) + .unwrap() + .prepare_restore( + &SnapshotRestoreRequest { + directory: directory.into(), + target, + recorded: recorded.clone(), + expected: recorded, + paths: None, + }, + 1024 * 1024, + 8 * 1024 * 1024, + ) + .await + .unwrap(); + let store = crate::copy::Store::open( + &data, + directory, + crate::copy::Options { + max_file: 1024 * 1024, + max_total: 8 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }, + ) + .unwrap(); + assert_eq!(store.preview_git(&prepared).unwrap().len(), 1); + std::fs::write(global, "# newly created global config\n").unwrap(); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!(std::fs::read(work.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index 4ca4cf1fb..df7bb94b1 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -31,6 +31,7 @@ pub(crate) struct CaptureSettings { #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] pub(crate) struct IgnoreInputs { + pub config: crate::copy::config_inputs::ConfigInputs, pub info: PathBuf, pub global: Option, pub case_insensitive: bool, @@ -56,50 +57,33 @@ pub(crate) struct PreparedRestore { impl Shadow { pub(crate) async fn ignore_inputs(&self) -> Result { + let (config, origins) = self.config_inputs().await?; let default_global = std::env::var_os("XDG_CONFIG_HOME") .filter(|value| !value.is_empty()) .map(|path| PathBuf::from(path).join("git/ignore")) .unwrap_or_else(|| PathBuf::from("~/.config/git/ignore")); - let configured = self - .run_bounded( - &[ - "config", - "--null", - "--path", - "--default", - default_global - .to_str() - .ok_or("Invalid global ignore path")?, - "--get", - "core.excludesfile", - ], - 8192, - ) - .await?; - let global = std::str::from_utf8( - configured - .strip_suffix(&[0]) - .ok_or("Invalid global ignore setting")?, - ) - .map_err(|_| "Non-UTF-8 global ignore path is unsupported")?; - let global = (!global.is_empty()).then(|| self.worktree.join(global)); - let ignore_case = self - .run_bounded( - &[ - "config", - "--bool", - "--default", - "false", - "--get", - "core.ignorecase", - ], - 16, - ) - .await?; + let raw = match origins.selected("core.excludesfile")? { + Some(value) => value.ok_or("Git ignore path lacks a value")?, + None => default_global + .to_str() + .ok_or("Invalid global ignore path")?, + }; + let global = self + .expand_config_paths(&[raw]) + .await? + .pop() + .ok_or("Missing Git ignore expansion")?; + let global = (!global.as_os_str().is_empty()).then(|| self.worktree.join(global)); + let raw = origins + .selected("core.ignorecase")? + .map_or("false", |value| value.unwrap_or("true")); + let ignore_case = self.config_boolean("core.ignorecase", raw).await?; + self.verify_config_inputs(&config, &origins).await?; Ok(IgnoreInputs { + config, info: self.git_dir.join("info/exclude"), global, - case_insensitive: ignore_case == b"true\n", + case_insensitive: ignore_case, private_roots: self.private_roots.clone(), settings: self.capture_settings.clone(), }) diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index a52131be4..b41d85e6a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2560,3 +2560,95 @@ disabled snapshots and ordinary no-op receipts. These cases preserve unmanaged file-copy and legacy Git behavior; the new SHA-256 fixtures above separately cover actual repository-format discovery and scoped materialization. No managed Git runtime activation or full goal acceptance is inferred from these results. + + +## Capture-bound Git configuration locator observations (2026-11-07) + +A regression against the preceding local production implementation at 4d801312 +prepared a valid native Git restore, then changed the private config's +core.excludesfile from one empty fixture file to another. The restore proceeded: +only resolved ignore inputs were retained, so changing the selecting configuration +after preparation escaped revalidation. The new test failed before implementation +and passes now, with unchanged source bytes and no published journal. This is a +local-head regression comparison, not attribution to unchanged main. + +Supported policy collection now obtains bounded origin metadata and selecting +root candidates, including nonexistent Git-reported system/global config files, +the shadow config/config.worktree/HEAD, explicit config environment and include +targets resolved relative to their origin files. Configuration files are regular, +size-bounded stable observations with content digest, native generation, canonical +route and parent identity. Environment values enter only a digest; full configuration file contents and +cleartext environment values are not retained in the handoff or checkpoint tags. +Resolved policy paths remain necessary private handoff metadata. +Physical worktree/private-directory identities bind Git-directory conditions. +Root/input state is rechecked around resolution and in native inventory before +intent publication, without starting Git inside the synchronous filesystem worker. + +Review corrected a value-selection race: querying settings independently of their +origin stream could choose a different include during resolution. Values now come +from the bounded inventory itself. Git expands explicit supplied overrides, with +include paths batched under argument/output limits and literal NUL framing. It +cannot silently reselect a different setting. Unsupported named-user home expansion +is refused rather than depending on an unbound account database. File alias checks +also bind canonical parent identity, including same-inode hardlink aliases. + +The policy fingerprint frame is now git-policy-capture-v2. Historical checkpoint +encodings remain readable; their old fingerprint cannot gain current config proof. +An old expected native checkpoint requires a fresh captured review instead of a +retroactive upgrade. This changes private policy evidence, not public API shapes. + +Native policy collection admits the app-created standalone private shadow with +unborn HEAD; common-dir/worktree/reftable substitutions, linked HEAD or loose/packed +branch refs are unsupported. Packed snapshot refs are allowed so ordinary Git GC +does not invalidate the policy merely by packing snapshots. Remote-URL conditional +include dependency closure and general alternate shadow reference storage are not +implemented: affected captures retain explicit unavailable policy/raw-state proof +and canonical legacy-readable checkpoints. They do not acquire native source, +checkout acknowledgement or cleanup authority. Full configuration/metadata ABA, +dependency and compatibility acceptance remains open; this is bounded optimistic +revalidation, not a claim of a globally atomic external-config snapshot. + +Git's own documentation describes +[system/global candidate paths even when absent](https://git-scm.com/docs/git-var#Documentation/git-var.txt-GITCONFIGGLOBAL), +and its [versioned config implementation](https://github.com/git/git/blob/v2.51.0/config.c) +shows the Git-directory/branch/remote dependencies that motivate these bounds. +Capability failure keeps legacy capture readable; no new production support floor +or managed Git source gate was selected. + +Shared bounded metadata execution now covers reads and process exit with a ten +second deadline. Timeout/output errors kill and await the exact subprocess before +returning. A real FIFO configuration fixture exercises timeout and subsequent +reader reuse. This does not extend that guarantee to every legacy capture/filter +subprocess; external filter effects remain a separate admission/compatibility gap. + +Verification on local macOS arm64: the locked full snapshot suite passed 153 tests +(113 unit, 18 file-copy integration, 22 Git integration), with four ignored helper +fixtures across binaries; the environment/global-root helpers execute through their +parent tests. The final focused capability case also passed after extending its +legacy-readability assertions. Real cases cover locator changes after preparation, +missing include/global creation, included file changes, conditional HEAD changes, +environment binding without cleartext values, file replacement/size limits, +symlink/hardlink retargeting, physical directory context, original private reference +layout and packed snapshot preservation. All-target locked snapshot Clippy passed +with Rust warnings denied. Full locked real CLI branch/recovery acceptance passed +all 17 cases, preserving actual kill/restart, keep/rollback, stage/clear and no-op +behavior. These CLI cases use unmanaged copy and legacy Git paths; they do not +accept managed Git runtime source integration. + +The code-review skill was applied across collectors, native observations, +subprocess cleanup, handoff/fingerprint, tests and spec deltas. It led to bound +value expansion, physical context and unsupported-layout/dependency refusals above. +Review verdict: approve this local gated metadata prerequisite, with no unresolved +critical/major finding for its exposed scope. Full managed Git activation still +requires the recorded dependency/ABA/filter, backend/runtime/checkout, migration, +crash/platform/client and reclamation acceptance; no M2–M8 or parent task closes. +Native Linux/Windows, TUI/App, full workspace, separate SDK runtime checks and +source-bound CI were not run for this continuation. Native C compilation reported +sandbox_init/sandbox_free_error deprecation warnings without origin attribution. + + +Final format/diff and generated SDK consistency checks passed. Pinned strict +OpenSpec validated 117 items; spec lint reported 0 errors and 25 warnings without +unchanged-main attribution. Public schema/SDK shapes remain unchanged. The +production managed-Git availability guard remains disabled, and no PR/push or +parent-task acceptance is made by this local prerequisite. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 0a8b9900f..36508338b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -650,3 +650,51 @@ replacement namespace. Comparing a copy checkpoint to a Git checkpoint SHALL refuse explicitly rather than return an empty difference or reinterpret IDs. This read-only compatibility grants no cross-backend restore/migration authority; that acceptance remains required before complete managed Git delivery. + + +### Capture-bound Git configuration locators + +The policy handoff SHALL bind the configuration roots, active origin files and +include targets that selected ignore locators, case behavior and includes. Roots +include Git-reported system/global candidates even when absent, the private shadow +config/config.worktree and HEAD, and explicit environment config paths. Included +paths SHALL be expanded by Git, interpreted relative to their origin file and +bounded together with origin metadata. Relevant inherited environment SHALL be +bound by a digest; persistent handoff evidence SHALL NOT retain full configuration +file contents or cleartext environment values. Resolved policy paths remain part +of the private handoff. + +Resolution SHALL bracket selected-value queries with the same bounded origin/include +inventory and stable file observations. Native inventory SHALL revalidate exact +configuration file state and environment before reading rules and again before +intent publication, without spawning Git from the synchronous filesystem worker. +Changed locators, missing-file creation, include changes and HEAD-dependent policy +changes SHALL refuse before source effects. Configuration origins grant read-only +policy evidence, never source mutation or checkout acknowledgement authority. +Unsupported metadata/capabilities retain explicit unavailable native proof and +legacy-readable capture behavior. A new policy fingerprint frame SHALL distinguish +this proof; old checkpoints SHALL NOT be retroactively upgraded. + + +Physical worktree/private-Git-directory identities and canonical routes SHALL be +bound because conditional Git-directory includes depend on that context. Selected +values SHALL come from the bound origin stream; Git path/boolean expansion SHALL +use explicit supplied overrides rather than reselect mutable settings in separate +queries. Include path expansion SHALL be batched and bounded. Named-user home +expansion without native account-database proof SHALL remain unavailable. Metadata +subprocesses SHALL have a bounded read/wait deadline and SHALL be killed and awaited +on timeout or output failure; this does not claim that every legacy capture/filter +subprocess has the same deadline or native source ownership. + + +The current prerequisite supports unconditional includes and Git-directory/branch +conditions in the application-created standalone private shadow with an unborn +HEAD. Its native checks decline common-dir/worktree/reftable substitutions, linked +HEADs and loose/packed branch references rather than inferring an unknown reference +store. Ordinary packed snapshot refs remain readable and may change through Git GC. +Remote-URL conditional dependencies and named-user home expansion lack complete +native dependency proof and SHALL retain legacy-readable captures with unavailable +native policy evidence. Full configuration/dependency/race and Git compatibility +acceptance remain required before managed source admission; these observations do +not claim an atomic snapshot of every external configuration input or independent +permission authority. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 7ba78660a..82af98a40 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -331,3 +331,10 @@ - **WHEN** the caller inspects changed paths, diffs or historical exclusions - **THEN** typed copy IDs SHALL select the existing Location-bound copy store without creating a missing namespace, and exclusions from copy and Git history SHALL remain reportable - **AND** comparisons between copy and Git IDs SHALL refuse explicitly without source effects or fabricated empty differences; read compatibility SHALL NOT authorize cross-backend restore or migration + + +#### Scenario: Git configuration locator changes after preparation +- **GIVEN** prepared native Git images bind the capture policy and resolved ignore inputs +- **WHEN** a selecting config root, include target, conditional-include HEAD or relevant environment changes after resolution +- **THEN** native inventory SHALL refuse before journal/source effects by revalidating bounded capture-bound configuration evidence without spawning Git in the filesystem worker +- **AND** absent root/include candidates SHALL be bound against later creation, unsupported origin/capability proof SHALL remain unavailable, and old policy fingerprints SHALL gain no retrospective proof diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 47d2bf0d8..134a6dcbe 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -417,3 +417,18 @@ by typed IDs through existing namespaces; mixed-backend comparison refuses. This does not admit managed Git source work or implement cross-backend restore/migration. These compatibility/acceptance requirements, configuration races, partial construction and reference-aware cleanup remain open with M2–M8 and parent tasks. + + +Git configuration locator continuation: supported native policy collection now +binds Git-reported root candidates (including absent files), active origins, +include targets, environment digest, exact file observations and physical directory +context. Selected values come from the bound origin inventory; batched explicit +path/boolean expansion cannot reselect a different value. Native planning rechecks +these inputs without Git subprocesses, using a new fingerprint frame that does not +upgrade old checkpoint proof. Metadata reads/waits have output/deadline limits and +kill/await cleanup. The application-created private reference-layout profile is +revalidated; unsupported topology, remote-URL conditions and named-user expansion +retain unavailable proof. This fixes the reproduced post-preparation locator change +but does not finish complete configuration/ABA/dependency, filter, actual Git +runtime/checkout, migration, crash/platform/client or reclamation acceptance. +Managed Git source admission and all parent tasks remain open and disabled. From 5be7ab93de4b03591062b9942980308dbb500f09 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 07:02:31 -0400 Subject: [PATCH 35/87] Retain typed copy image provenance in Git recovery journals --- crates/cyber-snapshot/src/copy/git.rs | 127 ++++++-- crates/cyber-snapshot/src/copy/git/tests.rs | 279 +++++++++++++++++- crates/cyber-snapshot/src/copy/git_images.rs | 99 +++++++ crates/cyber-snapshot/src/copy/mod.rs | 4 + crates/cyber-snapshot/src/copy/resolution.rs | 11 +- crates/cyber-snapshot/src/copy/restore.rs | 14 +- crates/cyber-snapshot/src/git/prepared.rs | 71 ++++- .../implementation.md | 87 ++++++ .../managed-recovery.md | 21 ++ .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 14 + 11 files changed, 692 insertions(+), 42 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/git_images.rs diff --git a/crates/cyber-snapshot/src/copy/git.rs b/crates/cyber-snapshot/src/copy/git.rs index 3ca581609..abd5a68de 100644 --- a/crates/cyber-snapshot/src/copy/git.rs +++ b/crates/cyber-snapshot/src/copy/git.rs @@ -2,7 +2,7 @@ use super::restore::{Change, RestoreContext}; use super::storage::{self, Result}; use super::{ENTRY_LIMIT, Entry, Store}; -use crate::git::prepared::{Image, PreparedRestore}; +use crate::git::prepared::{Image, ImageSource, PreparedRestore}; use cyber_server::runtime::{RestoreError, SnapshotWork}; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; @@ -14,29 +14,72 @@ use std::path::{Component, Path, PathBuf}; pub(super) struct GitContext { pub worktree: PathBuf, pub prefix: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sources: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ImageSources { + target: ImageSource, + recorded: ImageSource, + expected: ImageSource, +} +impl ImageSources { + fn for_ids(ids: [&str; 3]) -> Result> { + let kinds = ids.map(ImageSource::for_id); + let [target, recorded, expected] = kinds; + let (target, recorded, expected) = (target?, recorded?, expected?); + if expected != ImageSource::Git + || ids + .iter() + .filter(|id| !id.starts_with("copy:")) + .any(|id| id.len() != ids[2].len()) + { + return Err( + "Mixed Git context requires a common Git format and Git expected image".into(), + ); + } + Ok( + (target == ImageSource::Copy || recorded == ImageSource::Copy).then_some(Self { + target, + recorded, + expected, + }), + ) + } } impl RestoreContext { pub(super) fn validate(&self, location: &Path, version: u32) -> Result<()> { let ids = [&self.target, &self.recorded, &self.expected]; if let Some(git) = &self.git { - if !matches!(version, 4 | 5) + let sources = ImageSources::for_ids([&self.target, &self.recorded, &self.expected])?; + if !matches!(version, 4..=6) || !git.worktree.is_absolute() || git .worktree .components() .any(|part| matches!(part, Component::ParentDir | Component::CurDir)) || git.worktree.join(&git.prefix) != location - || !ids.iter().all(|id| git_id(id) && id.len() == ids[0].len()) + || match version { + 6 => sources.is_none() || git.sources != sources, + _ => { + sources.is_some() + || git.sources.is_some() + || !ids.iter().all(|id| git_id(id) && id.len() == ids[0].len()) + } + } { return Err("Invalid Git recovery snapshot context".into()); } if !git.prefix.is_empty() { super::tree::safe_path(&git.prefix)?; } - } else if !ids - .iter() - .all(|id| id.strip_prefix("copy:").is_some_and(storage::valid_digest)) + } else if version == 6 + || !ids + .iter() + .all(|id| id.strip_prefix("copy:").is_some_and(storage::valid_digest)) { return Err("Invalid recovery snapshot context".into()); } @@ -134,19 +177,6 @@ impl Store { &self, prepared: &PreparedRestore, ) -> std::result::Result<(Vec, RestoreContext), RestoreError> { - let context = RestoreContext { - target: prepared.target.id.clone(), - recorded: prepared.recorded.id.clone(), - expected: prepared.expected.id.clone(), - exclusions: prepared.exclusions.clone(), - git: Some(GitContext { - worktree: prepared.worktree.clone(), - prefix: prepared.prefix.clone(), - }), - }; - context - .validate(&self.root, 4) - .map_err(RestoreError::Failed)?; if prepared.directory != self.root { return Err(RestoreError::Failed( "Git handoff belongs to another native Location".into(), @@ -159,10 +189,55 @@ impl Store { )); } self.ensure_recovered().map_err(RestoreError::Failed)?; + let imported = self + .import_git_images(prepared) + .map_err(RestoreError::Failed)?; + let sources = ImageSources::for_ids([ + &prepared.target.id, + &prepared.recorded.id, + &prepared.expected.id, + ]) + .map_err(RestoreError::Failed)?; + let mut context = RestoreContext { + target: prepared.target.id.clone(), + recorded: prepared.recorded.id.clone(), + expected: prepared.expected.id.clone(), + exclusions: prepared.exclusions.clone(), + git: Some(GitContext { + worktree: prepared.worktree.clone(), + prefix: prepared.prefix.clone(), + sources, + }), + }; + context.exclusions.extend(imported.reports.iter().cloned()); + context + .validate( + &self.root, + if context + .git + .as_ref() + .is_some_and(|git| git.sources.is_some()) + { + 6 + } else { + 5 + }, + ) + .map_err(RestoreError::Failed)?; // Known image validation precedes any source effect. Full state scanning // and actual runtime lifetime integration remain required before activation. - for tree in [&prepared.target, &prepared.recorded, &prepared.expected] { + for tree in [ + imported.target.as_ref(), + imported.recorded.as_ref(), + &prepared.expected, + ] { + let source = ImageSource::for_id(&tree.id).map_err(RestoreError::Failed)?; for (path, image) in &tree.images { + if image.source != source { + return Err(RestoreError::Failed( + "Snapshot image provenance differs".into(), + )); + } image.verify().map_err(RestoreError::Failed)?; self.check_source_path(path).map_err(RestoreError::Failed)?; if !matches!(image.mode.as_str(), "100644" | "100755" | "120000") { @@ -184,11 +259,12 @@ impl Store { } } } - let paths: BTreeSet<_> = prepared + let paths: BTreeSet<_> = imported .target .images .keys() - .chain(prepared.recorded.images.keys()) + .chain(imported.recorded.images.keys()) + .filter(|path| !imported.excludes(prepared, path)) .chain(prepared.expected.images.keys()) .cloned() .collect(); @@ -205,9 +281,10 @@ impl Store { if !matches(current.as_ref(), prepared.expected.images.get(&path)) { return Err(RestoreError::Stale); } - let base = prepared.recorded.images.get(&path); - let want = prepared.target.images.get(&path); - if base == want + let base = imported.recorded.images.get(&path); + let want = imported.target.images.get(&path); + if imported.excludes(prepared, &path) + || base == want || prepared .allowed .as_ref() diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index 0dda634f6..85b5efc53 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -172,6 +172,160 @@ impl Repo { } } +#[tokio::test] +async fn mixed_copy_git_images_keep_original_ids_scope_exclusions_and_new_journal_authority() { + for (scoped, target_copy, recorded_copy) in [ + (false, true, true), + (true, true, true), + (false, true, false), + (true, true, false), + (false, false, true), + (true, false, true), + ] { + let repo = Repo::with_format("sha256"); + let location = if scoped { + repo.root.join("scope") + } else { + repo.root.clone() + }; + std::fs::create_dir_all(&location).unwrap(); + let mut store = repo.store(&location); + store.options.ignore = vec!["protected".into()]; + std::fs::write(location.join("file"), "before\n").unwrap(); + std::fs::write(location.join("protected"), "excluded before").unwrap(); + let target = if target_copy { + store.track().unwrap().tree + } else { + repo.track(&location).await + }; + std::fs::write(location.join("file"), "after\n").unwrap(); + let recorded = if recorded_copy { + store.track().unwrap().tree + } else { + repo.track(&location).await + }; + store.options.ignore.clear(); + std::fs::write(location.join("protected"), "new eligible owner bytes").unwrap(); + let expected = repo.track(&location).await; + let prepared = repo.prepare(&location, &target, &recorded, &expected).await; + let preview = store.preview_git(&prepared).unwrap(); + assert_eq!(preview.len(), 1); + assert_eq!(preview[0].file, if scoped { "scope/file" } else { "file" }); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); + assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before\n"); + assert_eq!( + std::fs::read(location.join("protected")).unwrap(), + b"new eligible owner bytes" + ); + let id = store.recovery().unwrap()[0].id.clone(); + let (dir, journal) = store.journal(&id).unwrap(); + assert_eq!(journal.version, 6); + let context = journal.context.as_ref().unwrap(); + assert_eq!( + (&context.target, &context.recorded, &context.expected), + (&target, &recorded, &expected) + ); + assert_eq!( + serde_json::to_value(context).unwrap()["git"]["sources"], + serde_json::json!({"target":if target_copy { "copy" } else { "git" },"recorded":if recorded_copy { "copy" } else { "git" },"expected":"git"}) + ); + assert!( + context + .exclusions + .iter() + .any(|value| value.contains("ignore: protected")) + ); + assert!(context.validate(&location, 5).is_err()); + let mut downgraded = journal.clone(); + downgraded.version = 5; + Store::record(&dir, &downgraded).unwrap(); + assert!(store.journal(&id).is_err()); + Store::record(&dir, &journal).unwrap(); + let mut mismatched = context.clone(); + mismatched + .git + .as_mut() + .unwrap() + .sources + .as_mut() + .unwrap() + .expected = ImageSource::Copy; + assert!(mismatched.validate(&location, 6).is_err()); + assert!( + !repo + .snapshots + .owned_restore_available(location.to_str().unwrap()) + ); + assert!(!repo.root.join(".git/index").exists()); + } +} + +#[tokio::test] +async fn mixed_image_import_refuses_corruption_foreign_manifests_stale_source_and_combined_budget() +{ + for fault in ["corrupt", "foreign", "stale", "budget", "packet"] { + let repo = Repo::new(); + let mut store = repo.store(&repo.root); + std::fs::write(repo.root.join("file"), "before\n").unwrap(); + let mut target = store.track().unwrap().tree; + std::fs::write(repo.root.join("file"), "after\n").unwrap(); + let recorded = store.track().unwrap().tree; + if fault == "corrupt" { + let tree = store.load(&target).unwrap(); + store + .objects + .write(&tree.files["file"].object, b"corrupt") + .unwrap(); + } + if fault == "foreign" { + let other = repo.root.join("foreign"); + std::fs::create_dir(&other).unwrap(); + let foreign = repo.store(&other); + let snapshot = foreign.track().unwrap().tree; + let digest = snapshot.strip_prefix("copy:").unwrap(); + let bytes = + storage::read(&foreign.manifests, digest, super::super::RECORD_LIMIT).unwrap(); + storage::atomic(&store.manifests, digest, &bytes).unwrap(); + target = snapshot; + } + let expected = repo.track(&repo.root).await; + let mut prepared = repo + .prepare(&repo.root, &target, &recorded, &expected) + .await; + if fault == "packet" { + let bytes = b"caller-supplied image".to_vec(); + prepared.target.images.insert( + "file".into(), + Image { + source: ImageSource::Copy, + mode: "100644".into(), + object: storage::digest(&bytes), + bytes, + }, + ); + } + if fault == "stale" { + std::fs::write(repo.root.join("file"), "later owner").unwrap(); + } + if fault == "budget" { + store.options.max_total = 18; + } + let error = store.restore_git_worker(prepared, None).unwrap_err(); + if fault == "budget" { + assert!(error.to_string().contains("handoff exceeds"), "{error}"); + } + assert_eq!( + std::fs::read(repo.root.join("file")).unwrap(), + if fault == "stale" { + b"later owner".as_slice() + } else { + b"after\n".as_slice() + } + ); + assert!(store.recovery().unwrap().is_empty()); + } +} + #[tokio::test] async fn native_git_import_restores_files_modes_creations_deletions_and_retains_original_ids() { let repo = Repo::new(); @@ -383,6 +537,7 @@ fn legacy_copy_context_keeps_its_encoding_and_cannot_be_reinterpreted_as_git() { assert_eq!(serde_json::to_value(&context).unwrap(), value); let mut invalid = context; invalid.git = Some(GitContext { + sources: None, worktree: "/fixture".into(), prefix: String::new(), }); @@ -395,16 +550,30 @@ async fn imported_git_preimages_support_native_keep_and_rollback_with_retained_g SnapshotBinding, SnapshotOrigin, SnapshotRecoveryAction as Action, SnapshotRecoveryConfirmation, }; - for action in [Action::Keep, Action::Rollback] { + for (action, mixed) in [ + (Action::Keep, false), + (Action::Rollback, false), + (Action::Keep, true), + (Action::Rollback, true), + ] { let repo = Repo::new(); + let store = repo.store(&repo.root); std::fs::write(repo.root.join("file"), b"target before").unwrap(); - let target = repo.track(&repo.root).await; + let target = if mixed { + store.track().unwrap().tree + } else { + repo.track(&repo.root).await + }; std::fs::write(repo.root.join("file"), b"recorded after").unwrap(); - let recorded = repo.track(&repo.root).await; + let recorded = if mixed { + store.track().unwrap().tree + } else { + repo.track(&repo.root).await + }; + let expected = repo.track(&repo.root).await; let prepared = repo - .prepare(&repo.root, &target, &recorded, &recorded) + .prepare(&repo.root, &target, &recorded, &expected) .await; - let store = repo.store(&repo.root); store.restore_git_worker(prepared, None).unwrap(); let id = store.recovery().unwrap()[0].id.clone(); let (dir, mut journal) = store.journal(&id).unwrap(); @@ -439,7 +608,7 @@ async fn imported_git_preimages_support_native_keep_and_rollback_with_retained_g .resolve_recovery(&confirmation, &|| Ok(()), None) .unwrap(); let (_, resolution) = store.journal(&receipt.resolution_id).unwrap(); - assert_eq!(resolution.version, 5); + assert_eq!(resolution.version, if mixed { 6 } else { 5 }); assert_eq!(resolution.context.as_ref().unwrap().target, target); assert_eq!( std::fs::read(repo.root.join("file")).unwrap(), @@ -1789,3 +1958,101 @@ async fn git_lifetime_port_keeps_scoped_locations_and_managed_checkout_completen .owned_restore_available(scope.to_str().unwrap()) ); } + +#[tokio::test] +async fn interrupted_mixed_native_replacement_reopens_and_rolls_back_only_retained_effects() { + use cyber_server::runtime::{ + SnapshotBinding, SnapshotOrigin, SnapshotRecoveryAction, SnapshotRecoveryConfirmation, + }; + let repo = Repo::new(); + let store = repo.store(&repo.root); + for file in ["a", "b"] { + std::fs::write(repo.root.join(file), "before").unwrap(); + } + let target = store.track().unwrap().tree; + for file in ["a", "b"] { + std::fs::write(repo.root.join(file), "after").unwrap(); + } + let recorded = store.track().unwrap().tree; + let expected = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &expected) + .await; + let (changes, context) = store.plan_git(&prepared).unwrap(); + let id = "restore_mixed_partial"; + let mut journal = store + .prepare_journal(changes, id, None, Some(context), None) + .unwrap(); + let stop = || { + if std::fs::read(repo.root.join("a")).unwrap() == b"before" { + Err("fixture stopped after first actual replacement".into()) + } else { + Ok(()) + } + }; + assert!( + store + .execute_journal(id, &mut journal, &stop, None) + .is_err() + ); + assert_eq!(std::fs::read(repo.root.join("a")).unwrap(), b"before"); + assert_eq!(std::fs::read(repo.root.join("b")).unwrap(), b"after"); + let (directory, mut journal) = store.journal(id).unwrap(); + assert_eq!(journal.version, 6); + assert!(!journal.completed); + // Native backend fixture only: no runtime/Store or checkout admission proof. + journal.origin = Some(SnapshotOrigin { + version: 1, + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + activity_id: "op_fixture".into(), + worktree_id: None, + binding: SnapshotBinding { + directory: store.root.clone(), + root_identity: store.root_id.clone(), + namespace_identity: storage::dir_identity(&store.journals).unwrap(), + journal_id: id.into(), + journal_identity: storage::dir_identity(&directory).unwrap(), + lock_identity: Some(store.lock_identity().unwrap()), + }, + }); + Store::record(&directory, &journal).unwrap(); + drop(directory); + drop(store); + let reopened = repo.store(&repo.root); + std::fs::write(repo.root.join("owner-file"), "later owner").unwrap(); + let action = SnapshotRecoveryAction::Rollback; + let review = reopened.recovery_review(id, action).unwrap(); + assert!(review.conflicts.is_empty()); + let confirmation = SnapshotRecoveryConfirmation { + journal_id: id.into(), + action, + confirmation: review.confirmation.unwrap(), + request_id: "mixed-partial-reopened".into(), + }; + let receipt = reopened + .resolve_recovery(&confirmation, &|| Ok(()), None) + .unwrap(); + assert_eq!( + reopened.journal(&receipt.resolution_id).unwrap().1.version, + 6 + ); + for file in ["a", "b"] { + assert_eq!(std::fs::read(repo.root.join(file)).unwrap(), b"after"); + } + assert_eq!( + std::fs::read(repo.root.join("owner-file")).unwrap(), + b"later owner" + ); + std::fs::write(repo.root.join("a"), "owner after resolution").unwrap(); + assert_eq!( + reopened + .resolve_recovery(&confirmation, &|| Ok(()), None) + .unwrap(), + receipt + ); + assert_eq!( + std::fs::read(repo.root.join("a")).unwrap(), + b"owner after resolution" + ); +} diff --git a/crates/cyber-snapshot/src/copy/git_images.rs b/crates/cyber-snapshot/src/copy/git_images.rs new file mode 100644 index 000000000..253934833 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/git_images.rs @@ -0,0 +1,99 @@ +//! Materialize old copy images only from their original Location-bound manifests. +use super::{ENTRY_LIMIT, Store, storage}; +use crate::git::prepared::{Image, ImageSource, PreparedRestore, PreparedTree}; +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) struct Imported<'a> { + pub target: Cow<'a, PreparedTree>, + pub recorded: Cow<'a, PreparedTree>, + pub excluded: BTreeSet, + pub reports: Vec, +} + +impl Store { + pub(super) fn import_git_images<'a>( + &self, + prepared: &'a PreparedRestore, + ) -> storage::Result> { + let mut remaining = self.options.max_total; + for tree in [&prepared.target, &prepared.recorded, &prepared.expected] { + for image in tree.images.values() { + remaining = remaining + .checked_sub(image.bytes.len() as u64) + .ok_or("Mixed image handoff exceeds byte limit")?; + } + } + let mut excluded = BTreeSet::new(); + let mut reports = vec![]; + let mut load = + |tree: &'a PreparedTree, label: &str| -> storage::Result> { + if ImageSource::for_id(&tree.id)? == ImageSource::Git { + return Ok(Cow::Borrowed(tree)); + } + if !tree.images.is_empty() { + return Err("Copy image authority requires its original manifest".into()); + } + let manifest = self.load(&tree.id)?; + for skipped in manifest.skipped { + let (_, path) = skipped + .split_once(": ") + .ok_or("Invalid historical copy exclusion")?; + super::tree::safe_path(path)?; + excluded.insert(path.to_owned()); + reports.push(format!("{label}: {skipped}")); + } + let mut images = BTreeMap::new(); + for (path, entry) in manifest.files { + super::tree::safe_path(&path)?; + if entry.length > self.options.max_file || images.len() >= ENTRY_LIMIT { + return Err("Copy journal image exceeds file/count limit".into()); + } + remaining = remaining + .checked_sub(entry.length) + .ok_or("Mixed image handoff exceeds byte limit")?; + images.insert( + path, + Image { + source: ImageSource::Copy, + mode: if entry.mode & 0o111 == 0 { + "100644" + } else { + "100755" + } + .into(), + bytes: self.bytes(&entry)?, + object: entry.object, + }, + ); + } + Ok(Cow::Owned(PreparedTree { + id: tree.id.clone(), + images, + })) + }; + let target = load(&prepared.target, "target")?; + let recorded = load(&prepared.recorded, "recorded")?; + Ok(Imported { + target, + recorded, + excluded, + reports, + }) + } +} + +impl Imported<'_> { + pub fn excludes(&self, prepared: &PreparedRestore, path: &str) -> bool { + let full = if prepared.prefix.is_empty() { + path.to_owned() + } else { + format!("{}/{path}", prepared.prefix) + }; + crate::git::history::excludes(&prepared.historical_excluded, &full) + || self + .excluded + .iter() + .any(|excluded| path == excluded || path.starts_with(&format!("{excluded}/"))) + } +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 4fb559527..7ccb2a80d 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -1,7 +1,11 @@ //! Location-bound native recovery journals and private non-Git BLAKE3 snapshots. pub(crate) mod config_inputs; +pub(crate) fn object_digest(bytes: &[u8]) -> String { + storage::digest(bytes) +} mod git; pub(crate) mod git_capture; +mod git_images; pub(crate) mod git_rules; mod inventory; mod links; diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs index 32142d361..6f159e9fd 100644 --- a/crates/cyber-snapshot/src/copy/resolution.rs +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -176,7 +176,16 @@ fn immutable_digest(record: &Journal) -> Result { match &record.backend { None => Ok(legacy), Some(backend) => Ok(storage::digest( - &serde_json::to_vec(&("snapshot-journal-v5", backend, legacy)).map_err(fail)?, + &serde_json::to_vec(&( + if record.version == 6 { + "snapshot-journal-v6" + } else { + "snapshot-journal-v5" + }, + backend, + legacy, + )) + .map_err(fail)?, )), } } diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 9de79130d..7026f1f5e 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -32,6 +32,7 @@ pub(super) struct Journal { // only the worker and cannot acquire that historical proof retroactively. // Version 4 binds Git context. Version 5 adds explicit backend and link kinds; // older records retain their original file-only encoding and digest framing. + // Version 6 binds mixed copy/Git image provenance without relabeling IDs. pub version: u32, #[serde(default, skip_serializing_if = "Option::is_none")] pub backend: Option, @@ -99,14 +100,14 @@ impl Store { .map_err(fail)?; if !matches!( (record.version, record.origin.is_some()), - (1, false) | (2, true) | (3, _) | (4, _) | (5, _) + (1, false) | (2, true) | (3, _) | (4, _) | (5, _) | (6, _) ) || (record.version == 4 && !record .context .as_ref() .is_some_and(|context| context.git.is_some())) || (record.version < 5 && record.backend.is_some()) - || (record.version == 5 + || (record.version >= 5 && match &record.backend { Some(JournalBackend::Git) => !record .context @@ -118,6 +119,7 @@ impl Store { .is_some_and(|context| context.git.is_some()), None => true, }) + || (record.version == 6 && record.backend != Some(JournalBackend::Git)) || record.location != self.root || record.root != self.root_id || (record.version < 5 @@ -553,6 +555,12 @@ impl Store { resolution.outcome = Some(resolution.review.budget()); } let version = if context + .as_ref() + .and_then(|context| context.git.as_ref()) + .is_some_and(|git| git.sources.is_some()) + { + 6 + } else if context .as_ref() .is_some_and(|context| context.git.is_some()) || changes.iter().any(|change| { @@ -571,7 +579,7 @@ impl Store { }; let mut journal = Journal { version, - backend: (version == 5).then(|| { + backend: (version >= 5).then(|| { if context .as_ref() .is_some_and(|context| context.git.is_some()) diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index df7bb94b1..2e746a459 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct Image { + pub source: ImageSource, pub mode: String, pub object: String, pub bytes: Vec, @@ -13,10 +14,43 @@ pub(crate) struct Image { impl Image { pub(crate) fn verify(&self) -> Result<(), String> { - images::verify(&self.object, &self.bytes) + match self.source { + ImageSource::Git => images::verify(&self.object, &self.bytes), + ImageSource::Copy if self.object == crate::copy::object_digest(&self.bytes) => Ok(()), + ImageSource::Copy => Err("Copy image does not match its object identity".into()), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ImageSource { + Git, + Copy, +} + +impl ImageSource { + pub(crate) fn for_id(id: &str) -> Result { + if let Some(digest) = id.strip_prefix("copy:") { + if digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Ok(Self::Copy); + } + } else if matches!(id.len(), 40 | 64) + && id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Ok(Self::Git); + } + Err("Invalid snapshot image source identity".into()) } } +#[derive(Clone)] pub(crate) struct PreparedTree { pub id: String, /// Paths relative to the exact Location, rather than the Git worktree root. @@ -50,6 +84,7 @@ pub(crate) struct PreparedRestore { /// The expected snapshot's exact worktree-relative exclusions, not globs. /// Historical exclusions constrain mutation, but not the current-state check. pub expected_excluded: Vec, + pub historical_excluded: Vec, pub allowed: Option>, pub ignore: IgnoreInputs, pub expected_policy: Option, @@ -108,17 +143,14 @@ impl Shadow { return Err("Git image handoff belongs to another Location".into()); } let ids = [&request.target, &request.recorded, &request.expected]; - let mut excluded = BTreeSet::new(); - for id in ids { - excluded.extend(self.recorded_exclusions(id).await?.unwrap_or_default()); + if ImageSource::for_id(&request.expected)? != ImageSource::Git { + return Err("Mixed native restore requires a fresh Git expected checkpoint".into()); } + let (excluded, exclusions) = self.git_image_exclusions(ids).await?; let expected_excluded = self .recorded_exclusions(&request.expected) .await? .unwrap_or_default(); - let exclusions = self - .historical_exclusions(&ids.map(|id| id.to_owned())) - .await?; let allowed = request .paths .as_ref() @@ -133,6 +165,13 @@ impl Shadow { let mut trees = Vec::new(); for (index, id) in ids.into_iter().enumerate() { let mut entries = BTreeMap::new(); + if ImageSource::for_id(id)? == ImageSource::Copy { + trees.push(PreparedTree { + id: id.clone(), + images: entries, + }); + continue; + } for (path, (mode, object)) in self.native_image_catalog(id).await?.iter() { let omitted = if index == 2 { history::excludes(&expected_excluded, path) @@ -160,6 +199,7 @@ impl Shadow { entries.insert( relative(prefix, path)?, Image { + source: ImageSource::Git, mode: mode.clone(), object: object.clone(), bytes, @@ -181,11 +221,28 @@ impl Shadow { expected: trees.next().ok_or("Missing expected images")?, exclusions, expected_excluded, + historical_excluded: excluded.into_iter().collect(), allowed, ignore, expected_policy, }) } + + async fn git_image_exclusions( + &self, + ids: [&String; 3], + ) -> Result<(BTreeSet, Vec), String> { + let mut excluded = BTreeSet::new(); + let mut git_ids = vec![]; + for id in ids { + if ImageSource::for_id(id)? == ImageSource::Copy { + continue; + } + git_ids.push(id.clone()); + excluded.extend(self.recorded_exclusions(id).await?.unwrap_or_default()); + } + Ok((excluded, self.historical_exclusions(&git_ids).await?)) + } } fn relative(prefix: &str, path: &str) -> Result { diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index b41d85e6a..c657e41f5 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2652,3 +2652,90 @@ OpenSpec validated 117 items; spec lint reported 0 errors and 25 warnings withou unchanged-main attribution. Public schema/SDK shapes remain unchanged. The production managed-Git availability guard remains disabled, and no PR/push or parent-task acceptance is made by this local prerequisite. + + +## Mixed copy/Git historical image materialization (2026-11-07) + +The native Git planner now accepts retained copy target and/or recorded checkpoints +with a fresh raw Git expected checkpoint. Immutable Git preparation retains typed +copy placeholders without attempting Git object lookup for copy IDs. Inside the +synchronous native store, original copy manifests are loaded under the existing +namespace lock, including their original Location/root binding and immutable tree +digest. Original object bytes are checked against BLAKE3 identities; Git images +retain typed SHA-1/SHA-256 verification. These image kinds are explicit, including +when both content hashes happen to be 64 characters. IDs remain unchanged: a copy +checkpoint is never relabeled as a Git checkpoint or given an invented Git blob. +Git image buffers are borrowed; copy buffers are materialized under shared total +and per-file/count limits. Limits conservatively cover the full prepared/imported +image set; this is not complete selected-path/large-image migration acceptance. + +Git worktree/prefix and exact native Location still bind the handoff. Historical +exclusions from copy and Git constrain target/recorded mutation; expected eligible +images remain in the complete current-state check even when a historical exclusion +protects them from mutation. Originally skipped files that are now eligible remain +unchanged. Copy manifests cannot supply caller-populated image packets instead of +actual manifest authority. Corrupt objects and copied-in foreign manifests refuse +before intent publication or source effects. + +A new version-6 Git context records target/recorded/expected image-source kinds and +original IDs. The physical journal backend remains Git; its native payload objects +still use the existing BLAKE3 CAS. Version-6 validation requires a Git expected +checkpoint, at least one copy historical input, consistent Git ID widths and exact +kind/ID correspondence. Version-4/5 encodings omit the optional section and retain +all-Git interpretation; annotation or version downgrades refuse instead of adopting +mixed authority. Version-6 immutable digest framing is distinct, and new recovery +intents preserve the context and original identifiers. Existing older digest +regressions still pass. + +Native backend regressions cover all three mixed source combinations at both root +and nested Locations, using an actual SHA-256 repository and retained copy stores. +They check preview/restore, original IDs, typed annotations, historical exclusions, +user-index preservation and old-version/mismatched-kind refusal. Separate cases +refuse corrupt copy objects, a valid foreign manifest copied into the current +namespace, stale source bytes and combined budget overflow without a journal or +source modification. The existing native keep/rollback test now exercises both +all-Git and mixed version-6 contexts, preserving unrelated edits and never replaying +source changes for a repeated request. + +A further real-filesystem case stops execution after the first actual namespace +replacement, verifies the second file is untouched, drops/reopens the native Store, +and rolls back only proven retained effects. Unrelated owner files and edits made +after resolution survive; repeated lookup returns the same receipt without replay. +This is injected interruption and reopened native storage, not an OS kill/restart +or runtime-bound checkout test. Synthetic fixture origins grant no runtime Store +pair or actual worker-lifetime acknowledgement authority. + +The cognitive-complexity skill was consulted. Its dispatcher does not support +Rust; native Clippy identified preparer complexity 17 against the repository's +limit of 15. Extracting historical Git exclusion collection into a small helper +resolved the diagnostic without allowances or policy changes. An incorrect FileDiff +field in the new test was also corrected before verification. + +The code-review skill was applied to source kinds/hashes, manifest/root validation, +combined limits, exclusion/stale-state separation, pre-effect unknown-journal +fencing, native preimages, version/digest compatibility, recovery/retry and tests. +Review verdict: approve this local gated backend continuation, with no unresolved +critical/major finding in that scope. Public client migration and runtime-owned Git +integration remain incomplete, alongside complete config/generated-rule-source, +filter and remaining compatibility, constructor/native kill/platform and GC +acceptance. No Git availability guard, parent task or issue is marked complete. + +Local macOS arm64 verification: the final full locked snapshot suite passed 156 +cases (116 unit, 18 file-copy integration, 22 Git integration), with four ignored +helper fixtures across binaries and no doc tests. The focused interrupted mixed +replacement case passed independently. All-target locked snapshot Clippy passed +with Rust warnings denied. The full locked 17-case real CLI branch/recovery suite +passed, preserving existing unmanaged copy/legacy Git kill/restart, keep/rollback, +stage/clear and no-op behavior. These CLI cases do not exercise public version-6 +migration because managed Git admission remains disabled. Public API shapes are +unchanged; no separate SDK, full workspace, TUI/App, Linux/Windows native execution +or source-bound CI is claimed for this continuation. Native C compilation emitted +sandbox_init/sandbox_free_error deprecation warnings without origin attribution. + + +Final-source denial coverage also rejects a caller-populated, internally +hash-consistent copy image packet instead of its original manifest; that focused +case passed after the full suite. Final format/diff, generated SDK consistency and +all-target snapshot Clippy checks passed. Pinned strict OpenSpec validated all 117 +items, and spec lint reported 0 errors/25 warnings without baseline attribution. +No PR/push or native Git source activation is made by this backend increment. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 36508338b..968a1e37e 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -698,3 +698,24 @@ native policy evidence. Full configuration/dependency/race and Git compatibility acceptance remain required before managed source admission; these observations do not claim an atomic snapshot of every external configuration input or independent permission authority. + + +### Mixed historical image provenance + +Native Git planning SHALL be able to use retained Location-bound copy target or +recorded images with a fresh raw Git expected checkpoint. It SHALL retain all +original checkpoint IDs and verify copy manifests/objects using their BLAKE3 +identity; no copy image SHALL be reinterpreted as a Git object or relabeled as a +Git checkpoint. The expected checkpoint SHALL retain complete current Git policy, +raw-image and namespace revalidation. Both backends' historical exclusions SHALL +constrain mutation while current expected images remain part of stale detection. + +A new version-6 Git journal context SHALL bind explicit target/recorded/expected +image-source kinds to the original IDs and common Git worktree/Location scope. +Version-4/5 Git contexts SHALL keep their original all-Git interpretation and byte +encoding; source annotations cannot grant mixed-image authority to an old record. +Version-6 immutable digest framing SHALL be distinct. Complete existing copy +manifest validation, byte/path/count budgets, scoped selection, native preimages, +conflict refusal, worker ownership, acknowledgement and recovery remain mandatory. +This image bridge alone SHALL NOT enable managed Git source admission or substitute +for runtime/client/migration and crash/retention acceptance. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 82af98a40..b314c7242 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -338,3 +338,10 @@ - **WHEN** a selecting config root, include target, conditional-include HEAD or relevant environment changes after resolution - **THEN** native inventory SHALL refuse before journal/source effects by revalidating bounded capture-bound configuration evidence without spawning Git in the filesystem worker - **AND** absent root/include candidates SHALL be bound against later creation, unsupported origin/capability proof SHALL remain unavailable, and old policy fingerprints SHALL gain no retrospective proof + + +#### Scenario: Native Git planning retains historical copy image provenance +- **GIVEN** a Location retains copy target/recorded checkpoints and a fresh raw Git expected checkpoint +- **WHEN** the native planner prepares restoration after backend routing changes +- **THEN** it SHALL verify each backend's original images and Location binding, retain namespaced IDs and historical exclusions, and revalidate complete current Git state before source effects +- **AND** a version-6 journal SHALL bind explicit per-image source kinds and distinct digest framing; old journal versions SHALL refuse those annotations rather than acquire new mixed-image authority diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 134a6dcbe..8cb2c37b5 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -432,3 +432,17 @@ retain unavailable proof. This fixes the reproduced post-preparation locator cha but does not finish complete configuration/ABA/dependency, filter, actual Git runtime/checkout, migration, crash/platform/client or reclamation acceptance. Managed Git source admission and all parent tasks remain open and disabled. + + +Mixed-image journal continuation: native Git preparation can retain copy target or +recorded IDs as typed deferred manifests with a fresh raw Git expected checkpoint. +The synchronous native store loads only its original Location-bound manifests, +verifies BLAKE3 objects, enforces combined image budgets, and applies both backends' +historical exclusions without hiding expected-state checks. A version-6 context +and digest bind per-image source kinds; old versions reject those annotations. +Native preview/restore and keep/rollback, including interrupted replacement after +reopening the namespace, have backend coverage. Runtime-owned Git callbacks, +client planning/routing, complete migration/limit combinations, config/generated +rule-source and filter compatibility, native kills/platforms and reference-aware +cleanup remain required. Managed Git source admission and M2–M8/parent tasks remain +open; backend fixture origins do not constitute runtime/checkout proof. From 4f0bfe84b02b866031fb8aeb2cdf275b3282ecd1 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 07:22:27 -0400 Subject: [PATCH 36/87] Bind generated Git excludes to source policy --- .../cyber-snapshot/src/copy/config_inputs.rs | 75 ++++- .../src/copy/generated_rules.rs | 268 ++++++++++++++++++ crates/cyber-snapshot/src/copy/git/tests.rs | 121 ++++++++ crates/cyber-snapshot/src/copy/git_rules.rs | 2 +- crates/cyber-snapshot/src/copy/mod.rs | 1 + crates/cyber-snapshot/src/git.rs | 34 ++- crates/cyber-snapshot/src/git/prepared.rs | 19 +- .../implementation.md | 76 +++++ .../managed-recovery.md | 15 + .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 14 + 11 files changed, 613 insertions(+), 19 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/generated_rules.rs diff --git a/crates/cyber-snapshot/src/copy/config_inputs.rs b/crates/cyber-snapshot/src/copy/config_inputs.rs index 21daf94bb..e83d47650 100644 --- a/crates/cyber-snapshot/src/copy/config_inputs.rs +++ b/crates/cyber-snapshot/src/copy/config_inputs.rs @@ -10,7 +10,7 @@ const TOTAL_LIMIT: usize = 8 * 1024 * 1024; #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] pub(crate) struct ConfigInputs { - environment: String, + environment: Option, files: BTreeMap>, contexts: BTreeMap, repository: Option, @@ -33,6 +33,16 @@ impl ConfigInputs { if environment_digest()? != environment { return Err("Git configuration environment changed".into()); } + let mut proof = Self::capture_files(paths)?; + if environment_digest()? != environment { + return Err("Git configuration environment changed".into()); + } + proof.environment = Some(environment); + Ok(proof) + } + + /// Generated source bytes have no environment-dependent interpretation. + pub(crate) fn capture_files(paths: Vec) -> storage::Result { let mut files = BTreeMap::new(); let mut bytes = 0; for path in paths { @@ -49,17 +59,61 @@ impl ConfigInputs { } files.insert(path, state); } - if environment_digest()? != environment { - return Err("Git configuration environment changed".into()); - } Ok(Self { - environment, + environment: None, files, contexts: BTreeMap::new(), repository: None, }) } + /// Reopen bytes only when they still match this observation. + pub(crate) fn bytes(&self, path: &Path) -> storage::Result> { + let expected = self.files.get(path).ok_or("Unbound Git policy input")?; + let (state, bytes) = read(path)?; + if &state != expected { + return Err("Git policy input changed while reading".into()); + } + Ok(bytes) + } + + pub(crate) fn fingerprint(&self) -> storage::Result { + Ok(storage::digest( + &serde_json::to_vec(self).map_err(storage::fail)?, + )) + } + + pub(crate) fn merge(&mut self, other: Self) -> storage::Result<()> { + if other.environment.as_ref().is_some_and(|environment| { + self.environment + .as_ref() + .is_some_and(|old| old != environment) + }) || other + .repository + .as_ref() + .is_some_and(|path| self.repository.as_ref() != Some(path)) + { + return Err("Git policy contexts disagree".into()); + } + self.environment = self.environment.clone().or(other.environment); + for (path, state) in other.files { + if self.files.get(&path).is_some_and(|old| old != &state) { + return Err("Git policy observations disagree".into()); + } + self.files.insert(path, state); + } + for (path, state) in other.contexts { + if self.contexts.get(&path).is_some_and(|old| old != &state) { + return Err("Git policy directory observations disagree".into()); + } + self.contexts.insert(path, state); + } + if self.files.len() > 1024 || self.contexts.len() > 4 { + return Err("Combined Git policy inputs exceed limits".into()); + } + self.verify() + } + pub(crate) fn bind_repository(&mut self, path: PathBuf) -> storage::Result<()> { private_repository(&path)?; self.repository = Some(path); @@ -104,10 +158,11 @@ impl ConfigInputs { } pub(crate) fn verify(&self) -> storage::Result<()> { - let mut current = Self::capture( - self.files.keys().cloned().collect(), - self.environment.clone(), - )?; + let paths = self.files.keys().cloned().collect(); + let mut current = match &self.environment { + Some(environment) => Self::capture(paths, environment.clone())?, + None => Self::capture_files(paths)?, + }; current.bind_contexts(self.contexts.keys().cloned().collect())?; if let Some(repository) = &self.repository { current.bind_repository(repository.clone())?; @@ -424,7 +479,7 @@ mod tests { let encoded = serde_json::to_string(&proof).unwrap(); assert!(!encoded.contains("fixture-secret-value-unique")); let mut changed = proof; - changed.environment = "0".repeat(64); + changed.environment = Some("0".repeat(64)); assert!(changed.verify().is_err()); } } diff --git a/crates/cyber-snapshot/src/copy/generated_rules.rs b/crates/cyber-snapshot/src/copy/generated_rules.rs new file mode 100644 index 000000000..818bd084f --- /dev/null +++ b/crates/cyber-snapshot/src/copy/generated_rules.rs @@ -0,0 +1,268 @@ +//! Bind generated shadow ignore rules to their original user-repository input. +use super::config_inputs::{ConfigInputs, environment_digest}; +use super::storage::{self, Result}; +use crate::git::prepared::CaptureSettings; +use cap_std::fs::Dir; +use std::path::Path; + +const LIMIT: u64 = 1024 * 1024; +const RECIPE: &str = "cyber-excludes.json"; +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Recipe { + version: u32, + source: String, + settings: CaptureSettings, + skipped: Vec, + output: String, +} + +fn source(common: &Path) -> Result<(ConfigInputs, Vec)> { + let path = common.join("info/exclude"); + let mut proof = ConfigInputs::capture_files(vec![path.clone()])?; + proof.bind_contexts(vec![common.to_path_buf()])?; + let bytes = proof.bytes(&path)?; + proof.verify()?; + Ok((proof, bytes)) +} + +fn render(mut bytes: Vec, settings: &CaptureSettings, skipped: &[String]) -> Result> { + if settings.ignore.len() > 4096 || skipped.len() > 4096 { + return Err("Generated Git ignore input count exceeds limit".into()); + } + bytes.push(b'\n'); + for pattern in &settings.ignore { + append(&mut bytes, pattern)?; + } + for path in skipped { + if path.contains(['\n', '\r', '\0']) { + return Err("Unsupported generated Git ignore path".into()); + } + append(&mut bytes, &format!("/{}", crate::git::escape(path)))?; + } + if bytes.len() as u64 > LIMIT { + return Err("Generated Git ignore bytes exceed limit".into()); + } + Ok(bytes) +} +fn append(bytes: &mut Vec, line: &str) -> Result<()> { + if line.len() > 8192 || (bytes.len() + line.len() + 1) as u64 > LIMIT { + return Err("Generated Git ignore bytes exceed limit".into()); + } + bytes.extend_from_slice(line.as_bytes()); + bytes.push(b'\n'); + Ok(()) +} + +pub(crate) fn write( + git_dir: &Path, + common: &Path, + settings: &CaptureSettings, + skipped: &[String], +) -> Result<()> { + let (proof, original) = source(common)?; + let bytes = render(original, settings, skipped)?; + let recipe = serde_json::to_vec(&Recipe { + version: 1, + source: proof.fingerprint()?, + settings: settings.clone(), + skipped: skipped.to_vec(), + output: storage::digest(&bytes), + }) + .map_err(storage::fail)?; + if recipe.len() as u64 > LIMIT { + return Err("Generated Git ignore recipe exceeds limit".into()); + } + let root = + Dir::open_ambient_dir(git_dir, cap_std::ambient_authority()).map_err(storage::fail)?; + let info = storage::shared_child(&root, "info")?; + proof.verify()?; + publish(&info, "exclude", &bytes)?; + publish(&info, RECIPE, &recipe)?; + proof.verify() +} + +fn publish(info: &Dir, name: &str, bytes: &[u8]) -> Result<()> { + // Keep generations stable across unchanged capture. Existing links or corrupt + // output are replaced through the directory descriptor, never followed. + if storage::read(info, name, LIMIT).is_ok_and(|current| current == bytes) { + return Ok(()); + } + storage::atomic(info, name, bytes) +} + +pub(crate) fn observe( + git_dir: &Path, + common: &Path, + settings: &CaptureSettings, +) -> Result { + let root = + Dir::open_ambient_dir(git_dir, cap_std::ambient_authority()).map_err(storage::fail)?; + let info = storage::existing_child(&root, "info", false)?; + let paths = vec![ + git_dir.join("info/exclude"), + git_dir.join("info").join(RECIPE), + ]; + let mut output = ConfigInputs::capture(paths.clone(), environment_digest()?)?; + output.bind_contexts(vec![git_dir.join("info")])?; + let encoded = storage::read(&info, RECIPE, LIMIT)?; + let recipe: Recipe = serde_json::from_slice(&encoded).map_err(storage::fail)?; + let (source, original) = source(common)?; + if recipe.version != 1 + || recipe.settings != *settings + || recipe.source != source.fingerprint()? + { + return Err("Generated Git ignore source or settings changed".into()); + } + let expected = render(original, settings, &recipe.skipped)?; + if recipe.output != storage::digest(&expected) + || storage::read(&info, "exclude", LIMIT)? != expected + || output.bytes(&paths[0])? != expected + || output.bytes(&paths[1])? != encoded + { + return Err("Generated Git ignore recipe or output changed".into()); + } + output.merge(source)?; + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + + struct Fixture { + _temp: tempfile::TempDir, + common: std::path::PathBuf, + shadow: std::path::PathBuf, + settings: CaptureSettings, + } + impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let common = base.join("common"); + let shadow = base.join("shadow"); + std::fs::create_dir_all(common.join("info")).unwrap(); + std::fs::create_dir(&shadow).unwrap(); + Self { + _temp: temp, + common, + shadow, + settings: CaptureSettings { + ignore: vec!["*.tmp".into()], + max_untracked: 65536, + }, + } + } + fn write(&self) { + write( + &self.shadow, + &self.common, + &self.settings, + &["big[1]".into()], + ) + .unwrap(); + } + fn proof(&self) -> Result { + observe(&self.shadow, &self.common, &self.settings) + } + } + + #[test] + fn unchanged_generation_retains_observations_and_missing_source_is_bound() { + let fixture = Fixture::new(); + fixture.write(); + let proof = fixture.proof().unwrap(); + fixture.write(); + assert_eq!(fixture.proof().unwrap(), proof); + proof.verify().unwrap(); + std::fs::write(fixture.common.join("info/exclude"), "").unwrap(); + assert!(proof.verify().is_err()); + assert!(fixture.proof().is_err()); + fixture.write(); + fixture.proof().unwrap().verify().unwrap(); + } + + #[test] + fn generation_recipe_output_settings_and_common_directory_are_bound() { + for kind in ["source", "output", "recipe", "settings", "common"] { + let mut fixture = Fixture::new(); + std::fs::write(fixture.common.join("info/exclude"), "*.secret\n").unwrap(); + fixture.write(); + let proof = fixture.proof().unwrap(); + match kind { + "source" => { + std::fs::write(fixture.common.join("info/exclude"), "changed\n").unwrap() + } + "output" => { + std::fs::write(fixture.shadow.join("info/exclude"), "changed\n").unwrap() + } + "recipe" => std::fs::write(fixture.shadow.join("info").join(RECIPE), "{}").unwrap(), + "settings" => fixture.settings.max_untracked += 1, + "common" => { + std::fs::rename(&fixture.common, fixture.common.with_extension("old")).unwrap(); + std::fs::create_dir_all(fixture.common.join("info")).unwrap(); + std::fs::write(fixture.common.join("info/exclude"), "*.secret\n").unwrap(); + } + _ => unreachable!(), + } + assert!(fixture.proof().is_err(), "{kind}"); + if kind != "settings" { + assert!(proof.verify().is_err(), "{kind}"); + } + } + } + + #[test] + fn generated_output_links_are_replaced_without_following_and_parent_links_refuse() { + let fixture = Fixture::new(); + fixture.write(); + let victim = fixture.common.join("victim"); + std::fs::write(&victim, "preserve").unwrap(); + for name in ["exclude", RECIPE] { + let path = fixture.shadow.join("info").join(name); + std::fs::remove_file(&path).unwrap(); + symlink(&victim, &path).unwrap(); + assert!(fixture.proof().is_err()); + fixture.write(); + assert!( + !std::fs::symlink_metadata(path) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(std::fs::read(&victim).unwrap(), b"preserve"); + } + std::fs::rename(fixture.shadow.join("info"), fixture.shadow.join("old-info")).unwrap(); + symlink(fixture.common.join("info"), fixture.shadow.join("info")).unwrap(); + assert!(write(&fixture.shadow, &fixture.common, &fixture.settings, &[]).is_err()); + assert!(fixture.proof().is_err()); + assert!(!fixture.common.join("info").join(RECIPE).exists()); + } + + #[test] + fn raw_source_bytes_are_preserved_and_unsupported_sources_are_bounded() { + let fixture = Fixture::new(); + let path = fixture.common.join("info/exclude"); + std::fs::write(&path, b"\xff\n").unwrap(); + fixture.write(); + assert!( + std::fs::read(fixture.shadow.join("info/exclude")) + .unwrap() + .starts_with(b"\xff\n") + ); + fixture.proof().unwrap().verify().unwrap(); + std::fs::write(&path, vec![0; LIMIT as usize + 1]).unwrap(); + assert!(write(&fixture.shadow, &fixture.common, &fixture.settings, &[]).is_err()); + std::fs::remove_file(&path).unwrap(); + assert!( + std::process::Command::new("mkfifo") + .arg(&path) + .status() + .unwrap() + .success() + ); + assert!(write(&fixture.shadow, &fixture.common, &fixture.settings, &[]).is_err()); + } +} diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index 85b5efc53..0bac7d567 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -11,6 +11,109 @@ struct Repo { snapshots: Arc, } +#[test] +fn unsupported_native_config_environment_keeps_ordinary_git_capture_available() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "copy::git::tests::multiline_environment_capture_child", + ]) + .env("GIT_CONFIG_COUNT", "1") + .env("GIT_CONFIG_KEY_0", "fixture.multiline") + .env("GIT_CONFIG_VALUE_0", "first\nsecond") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test] +#[ignore = "subprocess fixture with unsupported native configuration environment"] +async fn multiline_environment_capture_child() { + assert!(crate::copy::config_inputs::environment_digest().is_err()); + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), "canonical-readable").unwrap(); + let snapshot = repo + .snapshots + .track(repo.root.to_str().unwrap()) + .await + .unwrap() + .unwrap(); + assert!( + snapshot + .skipped + .iter() + .any(|value| value.contains("native Git policy/raw-state proof unavailable")) + ); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + assert_eq!( + shadow + .blob(&snapshot.tree, "file") + .await + .unwrap() + .unwrap() + .1, + b"canonical-readable" + ); +} + +#[tokio::test] +async fn non_utf8_user_excludes_preserve_canonical_capture_without_native_rule_authority() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), "legacy-readable").unwrap(); + std::fs::write(repo.root.join(".git/info/exclude"), b"\xff\n").unwrap(); + let snapshot = repo + .snapshots + .track(repo.root.to_str().unwrap()) + .await + .unwrap() + .unwrap(); + assert!( + snapshot + .skipped + .iter() + .any(|value| value.contains("native Git policy/raw-state proof unavailable")) + ); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + assert_eq!( + shadow + .blob(&snapshot.tree, "file") + .await + .unwrap() + .unwrap() + .1, + b"legacy-readable" + ); + assert!( + std::fs::read(shadow.git_dir.join("info/exclude")) + .unwrap() + .starts_with(b"\xff\n") + ); +} + +#[tokio::test] +async fn changed_user_excludes_after_preparation_refuses_native_source_effects() { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + assert_eq!(repo.track(&repo.root).await, recorded); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + std::fs::write(repo.root.join(".git/info/exclude"), "new-policy\n").unwrap(); + let store = repo.store(&repo.root); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); +} + #[tokio::test] async fn changed_git_config_locator_after_preparation_refuses_native_source_effects() { let repo = Repo::new(); @@ -1333,6 +1436,11 @@ async fn capture_bound_policy_rejects_application_ignore_and_size_setting_change std::fs::write(repo.root.join("file"), b"after").unwrap(); let recorded = repo.track(&repo.root).await; changed.store(true, Ordering::SeqCst); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + assert!(shadow.ignore_inputs().await.is_err()); + // A fresh capture regenerates rules for the new settings. The old + // checkpoint still cannot authorize native restore under this policy. + assert_ne!(repo.track(&repo.root).await, recorded); let prepared = repo .prepare(&repo.root, &target, &recorded, &recorded) .await; @@ -1618,6 +1726,19 @@ async fn sha256_linked_worktree_uses_its_common_repository_format_and_scoped_git .unwrap(); assert_eq!(paths, ["nested/file"]); assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); + let expected = repo.track(&location).await; + let prepared = repo + .prepare(&location, &recorded, &expected, &expected) + .await; + let store = repo.store(&location); + assert_eq!(store.preview_git(&prepared).unwrap().len(), 1); + std::fs::write( + repo.root.join(".git/info/exclude"), + "changed-common-policy\n", + ) + .unwrap(); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); assert_eq!(std::fs::read(linked.join(".git")).unwrap(), marker); assert!( !repo diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs index 051920039..b8c977c61 100644 --- a/crates/cyber-snapshot/src/copy/git_rules.rs +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -146,7 +146,7 @@ impl Rules { (left.source, &left.path, &left.hash).cmp(&(right.source, &right.path, &right.hash)) }); let bytes = serde_json::to_vec(&( - "git-policy-capture-v2", + "git-policy-capture-v3", &self.path, &self.inputs, bindings, diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 7ccb2a80d..7e2905748 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod config_inputs; pub(crate) fn object_digest(bytes: &[u8]) -> String { storage::digest(bytes) } +pub(crate) mod generated_rules; mod git; pub(crate) mod git_capture; mod git_images; diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index d3c0cda7e..13bc5c7da 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -204,7 +204,7 @@ impl Shadow { pub async fn track(&self, ignore: &[String], max_bytes: u64) -> Result { self.ensure_init().await?; let skipped = self.large_untracked(max_bytes).await; - self.write_excludes(ignore, &skipped)?; + self.write_excludes(ignore, max_bytes, &skipped)?; #[cfg(any(target_os = "macos", target_os = "linux"))] let capture_before = self.capture_git_state(ignore, max_bytes).await; let excluded = self.omitted().await?; @@ -254,11 +254,11 @@ impl Shadow { ignore: &[String], max_untracked: u64, ) -> Result { - let mut inputs = self.ignore_inputs().await?; - inputs.settings = prepared::CaptureSettings { + let settings = prepared::CaptureSettings { ignore: ignore.to_vec(), max_untracked, }; + let inputs = self.ignore_inputs_for(&settings).await?; let worktree = self.worktree.clone(); let prefix = if self.pathspec == "." { String::new() @@ -303,7 +303,31 @@ impl Shadow { } /// The user's `info/exclude`, `snapshots.ignore` and the skipped large files. - fn write_excludes(&self, ignore: &[String], skipped: &[String]) -> Result<(), String> { + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn write_excludes( + &self, + ignore: &[String], + max_untracked: u64, + skipped: &[String], + ) -> Result<(), String> { + crate::copy::generated_rules::write( + &self.git_dir, + &self.common_dir, + &prepared::CaptureSettings { + ignore: ignore.to_vec(), + max_untracked, + }, + skipped, + ) + } + + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + fn write_excludes( + &self, + ignore: &[String], + _max_untracked: u64, + skipped: &[String], + ) -> Result<(), String> { let mut text = std::fs::read_to_string(self.common_dir.join("info/exclude")).unwrap_or_default(); text.push('\n'); @@ -556,7 +580,7 @@ fn split_z(out: &[u8]) -> Vec { } /// Escape gitignore metacharacters so a path matches only itself. -fn escape(path: &str) -> String { +pub(crate) fn escape(path: &str) -> String { path.chars().fold(String::new(), |mut s, c| { if matches!(c, '*' | '?' | '[' | '\\' | '!' | '#') { s.push('\\'); diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index 2e746a459..539217eef 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -57,7 +57,8 @@ pub(crate) struct PreparedTree { pub images: BTreeMap, } -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub(crate) struct CaptureSettings { pub ignore: Vec, pub max_untracked: u64, @@ -92,7 +93,14 @@ pub(crate) struct PreparedRestore { impl Shadow { pub(crate) async fn ignore_inputs(&self) -> Result { - let (config, origins) = self.config_inputs().await?; + self.ignore_inputs_for(&self.capture_settings).await + } + + pub(super) async fn ignore_inputs_for( + &self, + settings: &CaptureSettings, + ) -> Result { + let (mut config, origins) = self.config_inputs().await?; let default_global = std::env::var_os("XDG_CONFIG_HOME") .filter(|value| !value.is_empty()) .map(|path| PathBuf::from(path).join("git/ignore")) @@ -114,13 +122,18 @@ impl Shadow { .map_or("false", |value| value.unwrap_or("true")); let ignore_case = self.config_boolean("core.ignorecase", raw).await?; self.verify_config_inputs(&config, &origins).await?; + config.merge(crate::copy::generated_rules::observe( + &self.git_dir, + &self.common_dir, + settings, + )?)?; Ok(IgnoreInputs { config, info: self.git_dir.join("info/exclude"), global, case_insensitive: ignore_case, private_roots: self.private_roots.clone(), - settings: self.capture_settings.clone(), + settings: settings.clone(), }) } diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index c657e41f5..de3cdc6f4 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2739,3 +2739,79 @@ case passed after the full suite. Final format/diff, generated SDK consistency a all-target snapshot Clippy checks passed. Pinned strict OpenSpec validated all 117 items, and spec lint reported 0 errors/25 warnings without baseline attribution. No PR/push or native Git source activation is made by this backend increment. + +## Generated Git exclude source binding (2026-11-07) + +A new real-Git regression reproduced a source-policy gap on local `5be7ab93`: +changing the user common repository's `info/exclude` after native preparation still +allowed source restoration. The regression failed its refusal assertion before the +production fix. The first invocation accidentally selected no tests with an +incomplete exact name; the fully qualified rerun executed and reproduced the gap. + +Supported native collection now verifies a bounded version-1 generation recipe +against fresh original source observations, exact application settings, skipped +literal paths and regenerated output bytes. The source locator is derived from +the trusted Shadow common directory, not deserialized metadata. Source-file absence +and common-directory identity are retained. The combined configuration proof also +retains generated output/recipe file observations and the shadow info directory. +The synchronous planner rechecks this combined proof without invoking Git. A new +`git-policy-capture-v3` frame does not promote older checkpoint policy evidence. + +Generation uses descriptor-relative atomic replacement beneath a no-follow info +directory. Existing output links are replaced without opening their target for +writes; unchanged output and recipe are not rewritten, preserving checkpoint +identity. Original source bytes are bounded and preserved, including non-UTF-8 +patterns, rather than silently dropping unreadable text. Unsupported/nonregular +source files fail generation explicitly. Windows retains its existing legacy +implementation and has no new native proof from this change. + +Review identified and corrected an initial compatibility regression: generated +source observation must not require a supported native configuration environment. +File-only observations now remain independent of environment interpretation, while +actual native configuration collection and merged proofs retain full environment +binding. An isolated multiline-configuration fixture verifies ordinary canonical +capture remains available with explicitly unavailable native proof. Another case +verifies non-UTF-8 exclude bytes retain canonical capture without native authority. +An unavailable Rust FIFO helper and a private-method reference in test fixtures +were corrected before final validation. Changed-settings coverage now checks early +recipe refusal and stale native policy after regeneration under fresh settings. + +Tests additionally cover changed user excludes before effects, stable repeated +checkpoint IDs, common-source changes for scoped SHA-256 linked worktrees, absent +source creation, physical common-directory replacement, recipe/output tampering, +settings mismatch, symlink-safe publication, parent-link refusal and over-limit/ +FIFO inputs. This is not a proof of user-index or large-untracked eligibility, +full configuration ABA/dependency/filter compatibility, actual managed Git runtime +ownership/checkout, complete crash/platform acceptance or reference-aware cleanup. +Managed Git availability and M2–M8/parent acceptance remain unchanged and open. + +The code-review skill was applied to all changed source, fixtures and spec deltas, +using the existing adapter architecture. Review checked bounded reads/rendering/ +parsing, trusted locator derivation, symlink/alias and directory replacement, +source-generation agreement, combined proof limits, native environment retention, +legacy/canonical compatibility, checkpoint stability and absence of new mutation/ +acknowledgement authority. The environment compatibility finding above is fixed +and regression-tested. No unresolved critical/major finding remains within this +local generated-source-binding scope. This does not accept the wider managed Git +or complete recovery feature. + +Final local macOS arm64 snapshot verification passed 163 cases (123 unit, 18 +file-copy integration, 22 Git integration), with five ignored subprocess fixtures +across test binaries and no doc tests. The isolated multiline environment fixture +is invoked by its passing parent. A lifetime fixture child emitted UnexpectedEof +during real-process death testing; both parent cases passed. Final all-target +locked snapshot Clippy passed with Rust warnings denied. The initial parallel +17-case real CLI suite passed 16 and failed one origin-before-source kill case at +its 30-second reservation-observation deadline. It did not reach recovery assertions; +no claim about the timeout's origin is made. The final-source serial CLI run subsequently passed all 17 cases in 204.49 seconds +without overlapping heavy filesystem fixtures. This supplies serial acceptance, +not an explanation or acceptance of the earlier concurrent observation timeout. + +Final format/diff and generated SDK consistency checks passed. Pinned strict +OpenSpec validated 117 items; spec lint reported 0 errors and 25 warnings without +unchanged-main attribution. Native C compilation emitted sandbox_init/ +sandbox_free_error deprecation warnings, and the CLI linker reported an oversized +unwind table warning; no origin attribution is made. This continuation did not run +native Linux/Windows, separate SDK runtime suites, full workspace, TUI/App or +source-bound CI. GitHub refresh found #93 open and no open PRs; fetched origin/main +remains db9e171e. No PR/push or parent-task acceptance is made by this local fix. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 968a1e37e..8f1101018 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -719,3 +719,18 @@ manifest validation, byte/path/count budgets, scoped selection, native preimages conflict refusal, worker ownership, acknowledgement and recovery remain mandatory. This image bridge alone SHALL NOT enable managed Git source admission or substitute for runtime/client/migration and crash/retention acceptance. + +### Generated Git exclude source binding + +Native policy proof SHALL bind the user common-repository `info/exclude` source, +including absence and physical common-directory identity, to the exact generated +shadow exclude bytes. A bounded versioned recipe SHALL retain a source observation +digest, capture settings, skipped literal paths and generated-byte digest. Source +locators SHALL derive from the trusted Shadow context, never decoded recipe paths. +Collection SHALL regenerate the bytes from fresh source observations and compare +both recipe and shadow output before granting proof. The synchronous native planner +SHALL revalidate these source, recipe and output observations before effects. +Unchanged generation SHALL preserve file observations and checkpoint identity; +publication SHALL use descriptor-relative replacement without following output +links. This does not prove user-index or large-untracked eligibility changes; +those and full configuration/filter compatibility still gate managed Git admission. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index b314c7242..d9466acf5 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -345,3 +345,10 @@ - **WHEN** the native planner prepares restoration after backend routing changes - **THEN** it SHALL verify each backend's original images and Location binding, retain namespaced IDs and historical exclusions, and revalidate complete current Git state before source effects - **AND** a version-6 journal SHALL bind explicit per-image source kinds and distinct digest framing; old journal versions SHALL refuse those annotations rather than acquire new mixed-image authority + +#### Scenario: Generated Git rules retain their original source +- **GIVEN** native Git preparation uses shadow rules generated from the user common repository's `info/exclude`, application settings and skipped paths +- **WHEN** the source file is changed or created after preparation, its common directory is replaced, or the generated recipe/output no longer matches fresh source observations +- **THEN** native planning SHALL refuse before new journal/source effects, including for linked worktrees and scoped Locations +- **AND** bounded recipe validation SHALL derive source locators from trusted context, preserve unchanged generation observations, and publish output without following existing file or parent links +- **AND** this proof SHALL NOT imply user-index/large-untracked eligibility or activate managed Git admission diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 8cb2c37b5..2fb812033 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -446,3 +446,17 @@ client planning/routing, complete migration/limit combinations, config/generated rule-source and filter compatibility, native kills/platforms and reference-aware cleanup remain required. Managed Git source admission and M2–M8/parent tasks remain open; backend fixture origins do not constitute runtime/checkout proof. + +Generated-rule source continuation: supported native policy now retains a bounded +version-1 generation recipe, original user common-repository exclude observations, +application settings, skipped literal names and exact output digest. Collection +regenerates and compares output, and native planning revalidates retained source, +recipe, output and common/info directory identity. Unchanged generation preserves +checkpoint identity; descriptor-relative output replacement does not follow file +or parent links. A new policy fingerprint frame leaves old evidence unpromoted. +This closes the reproduced post-preparation user-exclude change, including linked +worktrees, but does not prove user-index/large-untracked eligibility, complete Git +configuration/filter compatibility or actual runtime ownership. Managed Git source +admission, M2–M8 and parent tasks remain open. The next client milestone is typed +recovery review/confirm/lookup/acknowledgement on the implemented non-Git API, with +real restart/retry acceptance; it does not waive remaining Git/GC/platform gates. From bacade3a912ad4e391e8241d105bdeccd12b4254 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 07:38:33 -0400 Subject: [PATCH 37/87] Add typed snapshot recovery CLI commands --- crates/cyber-cli/src/commands/mod.rs | 1 + crates/cyber-cli/src/commands/recovery.rs | 536 ++++++++++++++++++ crates/cyber-cli/src/commands/sessions.rs | 8 +- crates/cyber-cli/tests/branches_cli.rs | 162 ++++-- docs/cli.md | 38 ++ docs/rewind.md | 68 ++- .../implementation.md | 77 +++ .../managed-recovery.md | 14 + .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 12 + 10 files changed, 870 insertions(+), 52 deletions(-) create mode 100644 crates/cyber-cli/src/commands/recovery.rs diff --git a/crates/cyber-cli/src/commands/mod.rs b/crates/cyber-cli/src/commands/mod.rs index 2a7c547bc..b288ae43a 100644 --- a/crates/cyber-cli/src/commands/mod.rs +++ b/crates/cyber-cli/src/commands/mod.rs @@ -30,6 +30,7 @@ pub mod plugin_config; pub mod plugins; pub mod pr; pub mod providers; +mod recovery; pub mod review; mod rewind; pub mod sandbox; diff --git a/crates/cyber-cli/src/commands/recovery.rs b/crates/cyber-cli/src/commands/recovery.rs new file mode 100644 index 000000000..4ade06581 --- /dev/null +++ b/crates/cyber-cli/src/commands/recovery.rs @@ -0,0 +1,536 @@ +//! Owner recovery through the existing Session-bound server protocol. +use clap::{Args, ValueEnum}; +use cyber_client::Client; +use cyber_server::runtime::{ + SnapshotRecovery, SnapshotRecoveryAction, SnapshotRecoveryConfirmation, SnapshotRecoveryImage, + SnapshotRecoveryResult, SnapshotRecoveryReview, SnapshotRecoverySelection, + SnapshotRecoveryState, +}; +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; + +use super::sessions::{api, clean, encode, session_id}; +use crate::{cli::GlobalArgs, error::CliError, output}; + +#[derive(Debug, Clone, Copy, ValueEnum)] +pub enum Action { + Keep, + Rollback, +} +impl From for SnapshotRecoveryAction { + fn from(action: Action) -> Self { + match action { + Action::Keep => Self::Keep, + Action::Rollback => Self::Rollback, + } + } +} +#[derive(Debug, Args)] +pub struct RecoveryArgs { + /// Session ID, exact title or unique ID prefix. + pub session: String, + /// Exact retained restore journal ID; omit to list journals. + #[arg(value_parser = journal_id, requires = "action")] + pub journal: Option, + /// Keep reviewed live files or roll back only proven restore effects. + #[arg(long, value_enum, requires = "journal")] + pub action: Option, + /// Apply the exact previously reviewed recovery fingerprint. + #[arg(long, value_parser = fingerprint, requires_all = ["journal", "request_id"])] + pub confirm: Option, + /// Stable request identity; retain it for lookup after interruption or a lost reply. + #[arg(long, value_parser = request_id, requires = "confirm")] + pub request_id: Option, +} +#[derive(Debug, Args)] +pub struct RequestArgs { + /// Session ID, exact title or unique ID prefix. + pub session: String, + /// Exact retained request ID from recovery confirmation. + #[arg(value_parser = request_id)] + pub request_id: String, + /// Reconcile native acknowledgement of the committed request; never replay source work. + #[arg(long)] + pub acknowledge: bool, +} + +fn journal_id(value: &str) -> Result { + if value.starts_with("restore_") + && value.len() <= 256 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + Ok(value.into()) + } else { + Err("expected an exact restore journal ID from sessions recovery".into()) + } +} +fn fingerprint(value: &str) -> Result { + if value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + Ok(value.into()) + } else { + Err("expected the 64-character lowercase fingerprint from recovery review".into()) + } +} +fn request_id(value: &str) -> Result { + if matches!(value, "." | "..") { + return Err("a dot path segment cannot be addressed as a recovery request; choose a stable named ID".into()); + } + if !value.is_empty() && value.len() <= 256 && !value.contains('\0') { + Ok(value.into()) + } else { + Err("recovery request ID must contain 1-256 bytes without NUL".into()) + } +} +fn data(reply: Value) -> Result { + serde_json::from_value(reply["data"].clone()) + .map_err(|error| CliError::runtime(format!("Invalid recovery response: {error}"))) +} +fn review_response( + reply: Value, + session: &str, + selection: &SnapshotRecoverySelection, +) -> Result { + let review: SnapshotRecoveryReview = data(reply)?; + if review.journal_id != selection.journal_id + || review.action != selection.action + || (review.confirmation.is_some() && review.origin.is_none()) + || review.origin.as_ref().is_some_and(|origin| { + origin.session_id != session || origin.binding.journal_id != selection.journal_id + }) + { + return Err(CliError::runtime( + "Recovery review does not match the selected Session and journal", + )); + } + Ok(review) +} +fn result_response( + reply: Value, + session: &str, + request: &str, +) -> Result { + let result: SnapshotRecoveryResult = data(reply)?; + if result.request.request_id != request + || result.original.session_id != session + || result.original.binding.journal_id != result.request.journal_id + { + return Err(CliError::runtime( + "Recovery reply does not match the selected Session and request", + )); + } + Ok(result) +} +fn lookup_hint(session: &str, request_id: &str) -> String { + if session + .chars() + .chain(request_id.chars()) + .any(char::is_control) + { + return format!( + "Inspect the retained request with cyber sessions recovery-request using the original IDs (JSON): Session {}, request {}; do not choose a new request ID to retry", + json!(session), + json!(request_id) + ); + } + // Shell quoting preserves unusual IDs exactly when copied from an error. + format!( + "Inspect the retained request with cyber sessions recovery-request -- {} {}; do not choose a new request ID to retry", + shell_words::quote(session), + shell_words::quote(request_id), + ) +} + +pub async fn run(client: &Client, args: RecoveryArgs, global: &GlobalArgs) -> Result<(), CliError> { + let id = session_id(client, &args.session).await?; + let base = format!("/sessions/{}/rewind/recovery", encode(&id)); + let Some(journal_id) = args.journal else { + let journals: Vec = data(client.get(&base).await.map_err(api)?)?; + return list(&journals, global); + }; + let action = args + .action + .ok_or_else(|| CliError::usage("Recovery requires --action"))? + .into(); + if let Some(confirmation) = args.confirm { + let request_id = args + .request_id + .ok_or_else(|| CliError::usage("Recovery confirmation requires --request-id"))?; + let hint = lookup_hint(&id, &request_id); + // The caller-provided identity stays unchanged on every transport attempt. + let body = SnapshotRecoveryConfirmation { + journal_id, + action, + confirmation, + request_id, + }; + let reply = client + .post(&format!("{base}/confirm"), json!(&body)) + .await + .map_err(|error| api(error).with_hint(&hint))?; + let result = result_response(reply, &id, &body.request_id) + .map_err(|error| error.with_hint(&hint))?; + if result.request != body { + return Err( + CliError::runtime("Recovery reply does not match the confirmed request") + .with_hint(hint), + ); + } + return result_output(&result, global, ResultOperation::Confirm); + } + let selection = SnapshotRecoverySelection { journal_id, action }; + let reply = client + .post(&format!("{base}/review"), json!(&selection)) + .await + .map_err(api)?; + let review = review_response(reply, &id, &selection)?; + review_output(&review, global) +} + +pub async fn request( + client: &Client, + args: RequestArgs, + global: &GlobalArgs, +) -> Result<(), CliError> { + let id = session_id(client, &args.session).await?; + let path = format!( + "/sessions/{}/rewind/recovery/requests/{}", + encode(&id), + encode(&args.request_id) + ); + let reply = if args.acknowledge { + client.post(&format!("{path}/acknowledge"), json!({})).await + } else { + client.get(&path).await + } + .map_err(|error| api(error).with_hint(lookup_hint(&id, &args.request_id)))?; + let result = result_response(reply, &id, &args.request_id) + .map_err(|error| error.with_hint(lookup_hint(&id, &args.request_id)))?; + let operation = if args.acknowledge { + ResultOperation::Acknowledge + } else { + ResultOperation::Lookup + }; + result_output(&result, global, operation) +} + +fn list(journals: &[SnapshotRecovery], global: &GlobalArgs) -> Result<(), CliError> { + if output::is_json(global.format) { + return output::json(&journals); + } + if journals.is_empty() { + println!("No retained restore journals."); + } + for journal in journals { + println!( + "{} filesystem_complete={} observed_applied={}/{}", + clean(&journal.id), + journal.filesystem_complete, + journal.applied.len(), + journal.paths.len() + ); + } + if !journals.is_empty() { + println!( + "Filesystem completion does not establish acknowledgement. Review an exact journal with --action keep or --action rollback." + ); + } + Ok(()) +} +fn action_name(action: SnapshotRecoveryAction) -> &'static str { + match action { + SnapshotRecoveryAction::Keep => "keep", + SnapshotRecoveryAction::Rollback => "rollback", + } +} +fn image(image: Option<&SnapshotRecoveryImage>) -> String { + match image { + None => "absent".into(), + Some(image) => format!( + "{:?} {} ({} bytes, mode {:o}, uid {}, gid {}, native {})", + image.kind, + clean(&image.hash), + image.bytes, + image.mode, + image.uid, + image.gid, + image + .native_identity + .as_deref() + .map(clean) + .unwrap_or_else(|| "unavailable".into()) + ), + } +} +fn literal(value: &str) -> String { + format!("{value:?}") +} +fn review_output(review: &SnapshotRecoveryReview, global: &GlobalArgs) -> Result<(), CliError> { + if output::is_json(global.format) { + return output::json(review); + } + println!( + "Recovery review: {} ({})", + clean(&review.journal_id), + action_name(review.action) + ); + println!( + "Filesystem complete: {}; acknowledged: {}", + review.filesystem_complete, review.acknowledged + ); + if let Some(origin) = &review.origin { + println!( + "Original Session: {}; worker: {}", + clean(&origin.session_id), + clean(&origin.activity_id) + ); + println!( + "Location: {}", + literal(&origin.binding.directory.to_string_lossy()) + ); + } else { + println!("Original worker identity: unavailable"); + } + for path in &review.paths { + println!( + "{} [{}] changes={}", + literal(&path.path), + clean(&path.state), + path.changes + ); + for (name, contents) in [ + ("current", &path.current), + ("before", &path.recorded_before), + ("after", &path.recorded_after), + ("retained", &path.retained), + ("proposed", &path.proposed), + ] { + println!(" {name}: {}", image(contents.as_ref())); + } + } + retained_paths("Historical exclusions", review.exclusions.as_deref()); + retained_paths( + "Protected created parents", + review.recorded_created_parents.as_deref(), + ); + for conflict in &review.conflicts { + println!("Conflict: {}", literal(conflict)); + } + println!("No files or conversation changed. External effects remain retained."); + match &review.confirmation { + Some(hash) => println!( + "To apply this review, repeat with --confirm {} --request-id ", + clean(hash) + ), + None => println!("Confirmation unavailable; retain the recovery evidence."), + } + Ok(()) +} +fn retained_paths(label: &str, paths: Option<&[String]>) { + match paths { + None => println!("{label}: unavailable historical evidence"), + Some([]) => println!("{label}: none"), + Some(paths) => { + for path in paths { + println!("{label}: {}", literal(path)); + } + } + } +} +enum ResultOperation { + Confirm, + Lookup, + Acknowledge, +} +fn result_output( + result: &SnapshotRecoveryResult, + global: &GlobalArgs, + operation: ResultOperation, +) -> Result<(), CliError> { + if output::is_json(global.format) { + output::json(result)?; + } else { + println!("Recovery request: {}", literal(&result.request.request_id)); + println!("Runtime state: {:?}", result.state); + println!( + "Native journal acknowledgement: {}", + result + .journal_acknowledged + .map_or("not observed by this response", |value| if value { + "observed" + } else { + "not acknowledged" + }) + ); + println!("The conversation is unchanged. External effects remain retained."); + } + let incomplete = match operation { + ResultOperation::Lookup => false, + ResultOperation::Confirm => result.state != SnapshotRecoveryState::Acknowledged, + ResultOperation::Acknowledge => { + result.state != SnapshotRecoveryState::Acknowledged + || result.journal_acknowledged != Some(true) + } + }; + if incomplete { + return Err( + CliError::runtime("Recovery is not acknowledged").with_hint(lookup_hint( + &result.original.session_id, + &result.request.request_id, + )), + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[test] + fn invalid_or_unreviewed_recovery_arguments_fail_before_startup() { + let hash = "a".repeat(64); + let oversized = "x".repeat(257); + for tail in [ + vec!["restore_one"], + vec!["--action", "keep"], + vec!["restore_one", "--action", "delete"], + vec!["restore_one", "--action", "keep", "--confirm", &hash], + vec!["restore_one", "--action", "keep", "--request-id", "one"], + vec![ + "restore_one", + "--action", + "keep", + "--confirm", + "yes", + "--request-id", + "one", + ], + vec!["../restore_one", "--action", "keep"], + vec![ + "restore_one", + "--action", + "keep", + "--confirm", + &hash, + "--request-id", + "..", + ], + vec![ + "restore_one", + "--action", + "keep", + "--confirm", + &hash, + "--request-id", + &oversized, + ], + ] { + let mut args = vec!["cyber", "sessions", "recovery", "ses_one"]; + args.extend(tail); + assert!(crate::cli::Cli::try_parse_from(&args).is_err(), "{args:?}"); + } + for tail in [ + vec![], + vec!["restore_one", "--action", "keep"], + vec![ + "restore_one", + "--action", + "rollback", + "--confirm", + &hash, + "--request-id", + "one / #?%", + ], + ] { + let mut args = vec!["cyber", "sessions", "recovery", "ses_one"]; + args.extend(tail); + crate::cli::Cli::try_parse_from(args).unwrap(); + } + } + + #[test] + fn malformed_and_foreign_replies_cannot_be_presented_as_the_requested_recovery() { + let selection = SnapshotRecoverySelection { + journal_id: "restore_one".into(), + action: SnapshotRecoveryAction::Keep, + }; + let origin = json!({"version":1,"session_id":"ses_one","session_created_id":"created","activity_id":"act_one","worktree_id":null, + "binding":{"directory":"/repo","root_identity":"root","namespace_identity":"namespace","journal_id":"restore_one","journal_identity":"journal"}}); + let mut review = json!({"data":{"journal_id":"restore_one","action":"keep","origin":null,"filesystem_complete":false,"acknowledged":false,"confirmation":null,"paths":[],"exclusions":null,"recorded_created_parents":null,"conflicts":["legacy proof unavailable"]}}); + review_response(review.clone(), "ses_one", &selection).unwrap(); + review["data"]["confirmation"] = "a".repeat(64).into(); + assert!(review_response(review.clone(), "ses_one", &selection).is_err()); + review["data"]["confirmation"] = Value::Null; + review["data"]["journal_id"] = "restore_foreign".into(); + assert!(review_response(review.clone(), "ses_one", &selection).is_err()); + review["data"]["journal_id"] = "restore_one".into(); + review["data"]["origin"] = origin.clone(); + assert!(review_response(review, "ses_foreign", &selection).is_err()); + let mut result = json!({"data":{"request":{"journal_id":"restore_one","action":"keep","confirmation":"a".repeat(64),"request_id":"one / #?%"},"original":origin,"resolution":null,"state":"pending","receipt":null,"journal_acknowledged":null}}); + result_response(result.clone(), "ses_one", "one / #?%").unwrap(); + assert!(result_response(result.clone(), "ses_one", "other").is_err()); + assert!(result_response(result.clone(), "ses_foreign", "one / #?%").is_err()); + result["data"]["original"]["binding"]["journal_id"] = "restore_foreign".into(); + assert!(result_response(result, "ses_one", "one / #?%").is_err()); + } + + #[test] + fn review_labels_preserve_distinct_paths_without_terminal_control_effects() { + assert_ne!(literal("one\ntwo"), literal("one two")); + let displayed = literal("one\n\x1b[31mtwo"); + assert!(!displayed.contains(['\n', '\x1b'])); + assert!(displayed.contains("\\n")); + } + + #[test] + fn request_lookup_never_accepts_confirmation_options_and_diagnostics_escape_controls() { + for option in ["--action", "--confirm", "--request-id"] { + assert!( + crate::cli::Cli::try_parse_from([ + "cyber", + "sessions", + "recovery-request", + "ses_one", + "request", + option, + "value" + ]) + .is_err() + ); + } + for tail in [vec![], vec!["--acknowledge"]] { + let mut args = vec![ + "cyber", + "sessions", + "recovery-request", + "ses_one", + "one / #?%", + ]; + args.extend(tail); + crate::cli::Cli::try_parse_from(args).unwrap(); + } + crate::cli::Cli::try_parse_from([ + "cyber", + "sessions", + "recovery-request", + "ses_one", + "--", + "-request", + ]) + .unwrap(); + assert!( + lookup_hint("ses_one", "-request").contains("recovery-request -- ses_one -request") + ); + let hint = lookup_hint("ses_one", "line\n\x1b[31m"); + assert!(!hint.contains(['\n', '\x1b'])); + assert!(hint.contains("\\n\\u001b")); + assert!(data::(json!({"data": {}})).is_err()); + } +} diff --git a/crates/cyber-cli/src/commands/sessions.rs b/crates/cyber-cli/src/commands/sessions.rs index 55cea1e10..d7484009b 100644 --- a/crates/cyber-cli/src/commands/sessions.rs +++ b/crates/cyber-cli/src/commands/sessions.rs @@ -76,6 +76,10 @@ pub enum SessionsCmd { #[arg(long, value_parser = super::rewind::fingerprint)] confirm: Option, }, + /// List retained restore journals, review keep/rollback, or confirm a reviewed recovery. + Recovery(super::recovery::RecoveryArgs), + /// Look up a retained recovery request; optionally reconcile its native acknowledgement. + RecoveryRequest(super::recovery::RequestArgs), /// Set or remove a branch label. Label { /// ID, exact title or unique ID prefix. @@ -199,6 +203,8 @@ async fn dispatch(client: &Client, cmd: SessionsCmd, global: &GlobalArgs) -> Res target, confirm, } => super::rewind::run(client, &session, &to, target, confirm, global).await, + SessionsCmd::Recovery(args) => super::recovery::run(client, args, global).await, + SessionsCmd::RecoveryRequest(args) => super::recovery::request(client, args, global).await, SessionsCmd::Other(_) => unreachable!("handled before connecting"), branch => dispatch_branch(client, branch, global).await, } @@ -406,7 +412,7 @@ async fn import( Ok(()) } -async fn session_id(client: &Client, wanted: &str) -> Result { +pub(super) async fn session_id(client: &Client, wanted: &str) -> Result { let found = resolve(client, wanted).await?; Ok(found["id"].as_str().unwrap_or_default().to_owned()) } diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 646e1bfd9..abcfabca8 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -752,26 +752,16 @@ fn recovery_review_binds_the_session_discloses_acknowledgement_and_starts_no_inf let requests = model.requests().len(); let original_messages = env.messages(&id); for action in ["keep", "rollback"] { - let body = json!({"journal_id":journal,"action":action}).to_string(); - let response = json(&env.cyber(&[ - "api", - "POST", - &format!("/sessions/{id}/rewind/recovery/review"), - "-d", - &body, + let review = json(&env.cyber(&[ + "sessions", "recovery", &id, journal, "--action", action, "--format", "json", ])); - let review = &response["data"]; assert_eq!(review["origin"]["session_id"], id); assert_eq!(review["acknowledged"], true); assert!(review["confirmation"].is_null()); assert!(!review["conflicts"].as_array().unwrap().is_empty()); assert!(review["exclusions"].is_array()); let foreign = env.cyber(&[ - "api", - "POST", - &format!("/sessions/{other}/rewind/recovery/review"), - "-d", - &body, + "sessions", "recovery", &other, journal, "--action", action, "--format", "json", ]); assert!(!foreign.status.success()); assert!( @@ -978,6 +968,83 @@ fn kill_before_source_admission(env: &Env, id: &str, first: &str, published: boo activity } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn recovery_confirm_cli(env: &Env, id: &str, request: &Value) -> Output { + env.cyber(&[ + "sessions", + "recovery", + id, + request["journal_id"].as_str().unwrap(), + "--action", + request["action"].as_str().unwrap(), + "--confirm", + request["confirmation"].as_str().unwrap(), + "--request-id", + request["request_id"].as_str().unwrap(), + "--format", + "json", + ]) +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn reviewed_recovery_cli(env: &Env, id: &str, action: &str) -> (String, Value) { + let catalog = json(&env.cyber(&["sessions", "recovery", id, "--format", "json"])); + let journals = catalog.as_array().unwrap(); + assert_eq!(journals.len(), 1); + assert_eq!(journals[0]["filesystem_complete"], false); + let journal = journals[0]["id"].as_str().unwrap().to_owned(); + let review = json(&env.cyber(&[ + "sessions", "recovery", id, &journal, "--action", action, "--format", "json", + ])); + let text = env.cyber(&["sessions", "recovery", id, &journal, "--action", action]); + let text = exits(&text, 0); + assert!( + text.contains("Recovery review:") + && text.contains("retained:") + && text.contains("proposed:") + && text.contains("Location:"), + "{text}" + ); + assert!( + review["conflicts"].as_array().unwrap().is_empty(), + "{review}" + ); + assert!( + text.contains("\"zz\\nfixture\""), + "review lost the literal newline path" + ); + (journal, review) +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_foreign_recovery_request_refused(env: &Env, request: &Value) { + let foreign = json(&env.cyber(&[ + "api", + "POST", + "/sessions", + "-d", + "{\"title\":\"foreign recovery request\"}", + ])); + let foreign_id = foreign["data"]["id"].as_str().unwrap(); + assert!( + !env.cyber(&[ + "sessions", + "recovery-request", + foreign_id, + request["request_id"].as_str().unwrap(), + "--format", + "json" + ]) + .status + .success() + ); + assert!( + !recovery_confirm_cli(env, foreign_id, request) + .status + .success() + ); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn managed_native_recovery_after_partial_kill(action: &str) { let model = scripted_model::ScriptedModel::start(reply); @@ -989,11 +1056,13 @@ fn managed_native_recovery_after_partial_kill(action: &str) { for index in 0..1280 { std::fs::write(repo.join(format!("bulk_{index:04}")), "before").unwrap(); } + std::fs::write(repo.join("zz\nfixture"), "before").unwrap(); let id = env.exec(None, "before snapshot"); let first = env.user_ids(&id)[0].clone(); for index in 0..1280 { std::fs::write(repo.join(format!("bulk_{index:04}")), "after").unwrap(); } + std::fs::write(repo.join("zz\nfixture"), "after").unwrap(); env.exec(Some(&id), "after snapshot"); let messages = kill_partial_restore(&env, &id, &first); let requests = model.requests().len(); @@ -1001,31 +1070,19 @@ fn managed_native_recovery_after_partial_kill(action: &str) { if action == "keep" { std::fs::write(repo.join("bulk_1279"), "later owner image").unwrap(); } - let catalog = json(&env.cyber(&["api", "GET", &format!("/sessions/{id}/rewind/recovery")])); - let journals = catalog["data"].as_array().unwrap(); - assert_eq!(journals.len(), 1); - assert_eq!(journals[0]["filesystem_complete"], false); - let journal = journals[0]["id"].as_str().unwrap(); - let review = json(&env.cyber(&[ - "api", - "POST", - &format!("/sessions/{id}/rewind/recovery/review"), - "-d", - &json!({"journal_id":journal,"action":action}).to_string(), - ]))["data"] - .clone(); - assert!( - review["conflicts"].as_array().unwrap().is_empty(), - "{review}" - ); - let request_id = format!("real-kill-{action}"); + let (journal, review) = reviewed_recovery_cli(&env, &id, action); + let request_id = format!("real-kill-{action}/ owner #?%"); let request = json!({"journal_id":journal,"action":action,"confirmation":review["confirmation"],"request_id":request_id}); - let confirm_path = format!("/sessions/{id}/rewind/recovery/confirm"); - let result = - json(&env.cyber(&["api", "POST", &confirm_path, "-d", &request.to_string()]))["data"] - .clone(); + let mut stale = request.clone(); + stale["confirmation"] = "0".repeat(64).into(); + let refused = recovery_confirm_cli(&env, &id, &stale); + assert_eq!(refused.status.code(), Some(1)); + assert!(String::from_utf8_lossy(&refused.stderr).contains("stale")); + assert!(String::from_utf8_lossy(&refused.stderr).contains("recovery-request")); + let result = json(&recovery_confirm_cli(&env, &id, &request)); assert_eq!(result["state"], "acknowledged"); assert_eq!(result["journal_acknowledged"], true); + assert_foreign_recovery_request_refused(&env, &request); assert_eq!(env.file("unrelated-owner").as_deref(), Some("owner data")); let expected = if action == "rollback" { "after" @@ -1039,7 +1096,7 @@ fn managed_native_recovery_after_partial_kill(action: &str) { requests, "recovery started inference" ); - assert_native_settlement_link(&env, journal, &result); + assert_native_settlement_link(&env, &journal, &result); assert_recovery_lookup_does_not_replay(&env, &id, &request_id, &request, &result, &messages); assert_eq!(model.requests().len(), requests); } @@ -1162,11 +1219,8 @@ fn assert_recovery_lookup_does_not_replay( messages: &[Value], ) { let repo = env.root.join("repo"); - let confirm_path = format!("/sessions/{id}/rewind/recovery/confirm"); std::fs::write(repo.join("bulk_0000"), "new owner edit after recovery").unwrap(); - let retry = - json(&env.cyber(&["api", "POST", &confirm_path, "-d", &request.to_string()]))["data"] - .clone(); + let retry = json(&recovery_confirm_cli(env, id, request)); assert_eq!(retry["state"], "acknowledged"); assert!( retry["journal_acknowledged"].is_null(), @@ -1179,18 +1233,19 @@ fn assert_recovery_lookup_does_not_replay( let mut changed = request.clone(); changed["confirmation"] = "b".repeat(64).into(); assert!( - !env.cyber(&["api", "POST", &confirm_path, "-d", &changed.to_string()]) - .status - .success(), + !recovery_confirm_cli(env, id, &changed).status.success(), "changed request content reused an existing owner" ); kill_registered_service(env); let lookup = json(&env.cyber(&[ - "api", - "GET", - &format!("/sessions/{id}/rewind/recovery/requests/{request_id}"), - ]))["data"] - .clone(); + "sessions", + "recovery-request", + id, + request_id, + "--format", + "json", + ])); + assert!(lookup["journal_acknowledged"].is_null()); assert_eq!(lookup["receipt"], result["receipt"]); assert_eq!( env.file("bulk_0000").as_deref(), @@ -1234,8 +1289,15 @@ fn reconcile_missing_journal_acknowledgement( let source = env.root.join("repo/bulk_0000"); std::fs::remove_file(&source).unwrap(); std::os::unix::fs::symlink(&target, &source).unwrap(); - let path = format!("/sessions/{id}/rewind/recovery/requests/{request_id}/acknowledge"); - let reconciled = json(&env.cyber(&["api", "POST", &path, "-d", "{}"]))["data"].clone(); + let reconciled = json(&env.cyber(&[ + "sessions", + "recovery-request", + id, + request_id, + "--acknowledge", + "--format", + "json", + ])); assert_eq!(reconciled["receipt"], result["receipt"]); assert_eq!(reconciled["journal_acknowledged"], true); assert!( diff --git a/docs/cli.md b/docs/cli.md index 1cb6ea52d..33235e1f8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -318,6 +318,8 @@ cyber sessions [OPTIONS] - [`tree`](#cyber-sessions-tree): The Session's user-message tree: `*` marks the active path - [`checkout`](#cyber-sessions-checkout): Make another branch active; the Session must be idle - [`rewind`](#cyber-sessions-rewind): Preview a rewind before a user message; apply only with its reviewed fingerprint +- [`recovery`](#cyber-sessions-recovery): List retained restore journals, review keep/rollback, or confirm a reviewed recovery +- [`recovery-request`](#cyber-sessions-recovery-request): Look up a retained recovery request; optionally reconcile its native acknowledgement - [`label`](#cyber-sessions-label): Set or remove a branch label - [`export`](#cyber-sessions-export): Write a Session as JSON (default), Markdown (--format md) or one self-contained HTML page (--format html): its info, messages, tool calls, goals and workflow runs - [`import`](#cyber-sessions-import): Import a JSON export (`cyber sessions export`) as a new Session in this Location, with fresh IDs and its attachments, which are verified before anything is written @@ -423,6 +425,42 @@ cyber sessions rewind [OPTIONS] --to - `--target `: Values: `code`, `conversation`, `both`. Default: `both`. - `--confirm `: Apply the exact previously reviewed preview; changed context requires a new preview. +## `cyber sessions recovery` + +List retained restore journals, review keep/rollback, or confirm a reviewed recovery + +```text +cyber sessions recovery [OPTIONS] [JOURNAL] +``` + +**Arguments:** + +- ``: Session ID, exact title or unique ID prefix. +- `[JOURNAL]`: Exact retained restore journal ID; omit to list journals. + +**Options:** + +- `--action `: Keep reviewed live files or roll back only proven restore effects. Values: `keep`, `rollback`. +- `--confirm `: Apply the exact previously reviewed recovery fingerprint. +- `--request-id `: Stable request identity; retain it for lookup after interruption or a lost reply. + +## `cyber sessions recovery-request` + +Look up a retained recovery request; optionally reconcile its native acknowledgement + +```text +cyber sessions recovery-request [OPTIONS] +``` + +**Arguments:** + +- ``: Session ID, exact title or unique ID prefix. +- ``: Exact retained request ID from recovery confirmation. + +**Options:** + +- `--acknowledge`: Reconcile native acknowledgement of the committed request; never replay source work. + ## `cyber sessions label` Set or remove a branch label diff --git a/docs/rewind.md b/docs/rewind.md index f87e358cd..6d4215deb 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -286,7 +286,7 @@ reports `cancelled_before_origin` or `cancelled_before_effects`. Created source directories, private payloads, native phase records and lifetime preparations remain protected for acknowledged safe cleanup. Do not edit records or remove evidence to force a retry. Partial acquisition recovery, owned Git -journaling, reclamation, typed recovery clients and broader native crash/platform +journaling, reclamation, preparation/TUI recovery clients and broader native crash/platform acceptance remain unfinished. @@ -340,3 +340,69 @@ follows that target while staging, exchanging or reading retained preimages. The local Git journal bridge supports typed native link fixtures, but ordinary owned Git recovery remains unavailable until its remaining lifecycle, full-state and platform acceptance gates are complete. + + +## Recovering an interrupted restore from the CLI + +List retained journals, then review an exact journal and action: + +```sh +cyber sessions recovery ses_example +cyber sessions recovery ses_example restore_example --action keep +cyber sessions recovery ses_example restore_example --action rollback --format json +``` + +Review does not change files or move the conversation. Text quotes paths and request IDs, escaping terminal controls without collapsing +distinct names. It shows original worker identity, +current/before/after/retained/proposed images, historical exclusions, +protected created parents, conflicts and acknowledgement status. Unavailable +historical evidence is shown explicitly. `keep` preserves the reviewed live files; +`rollback` reverses only effects proven to belong to that restore. Neither resumes +the failed rewind or undoes external tool effects. Conflicted or unverifiable +reviews have no confirmation fingerprint. + +Apply only a reviewed fingerprint, using a request ID you retain: + +```sh +cyber sessions recovery ses_example restore_example --action rollback \ + --confirm --request-id recovery-one +``` + +Malformed fingerprints and incomplete option combinations fail before service +startup. Request IDs contain 1–256 bytes without NUL; the CLI also refuses exact +`.` and `..`, which URL clients normalize as path segments. Other spaces, slashes +and URL punctuation are encoded without changing the request identity. Quote such +IDs in your shell. For a lookup ID beginning with `-`, use +`recovery-request -- ` so it is parsed as an ID. + +If interrupted or unsure whether the reply arrived, look up the same request: + +```sh +cyber sessions recovery-request ses_example recovery-one +cyber sessions recovery-request ses_example recovery-one --format json +``` + +Lookup reads the durable request without replaying files. Repeating the original +confirmation with the same request ID and identical content also performs lookup; +changed content refuses. Never choose a new ID to retry an uncertain outcome. +A successful lookup can report a pending or unknown outcome; inspect `state`. +Runtime `acknowledged` and native journal acknowledgement are separate fields. +`journal_acknowledged: null` means this lookup has not observed native metadata, +not that the metadata is absent. To explicitly reconcile an acknowledged durable +request whose native acknowledgement was interrupted: + +```sh +cyber sessions recovery-request ses_example recovery-one --acknowledge +``` + +This verifies the exact retained worker/checkout receipts and writes only native +acknowledgement metadata; it does not reinstall source files or change the +conversation. Confirmation/reconciliation exits 1 if the returned state does not +establish the requested acknowledgement. Lookup itself can return an unresolved +state with exit 0 because the read succeeded. Both commands support `--format json`. + +These commands expose the implemented managed file-copy recovery path on admitted +Linux/macOS hosts. Ordinary managed Git source recovery remains disabled pending +its remaining compatibility and runtime acceptance gates. Preparation cancellation +is still available through its existing API; typed preparation/TUI recovery, +reference-aware reclamation and complete native/platform acceptance remain open. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index de3cdc6f4..66f915f40 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2815,3 +2815,80 @@ unwind table warning; no origin attribution is made. This continuation did not r native Linux/Windows, separate SDK runtime suites, full workspace, TUI/App or source-bound CI. GitHub refresh found #93 open and no open PRs; fetched origin/main remains db9e171e. No PR/push or parent-task acceptance is made by this local fix. + +## Typed restore recovery CLI (2026-11-07) + +The approved `sessions recovery` command now lists retained journals without a +journal argument, reviews an exact journal/action, and confirms only with both the +reviewed lowercase fingerprint and caller-retained stable request ID. Clap refuses +malformed and incomplete combinations before server startup. The separate +`sessions recovery-request` command performs durable lookup; explicit +`--acknowledge` uses the existing metadata-only native reconciliation route. +No snapshot worker, journal format, permission boundary or backend availability +guard is changed. Existing API/SDK operations remain the source of recovery logic. + +Replies deserialize into the shared runtime contract. Reviews must match selected +Session/journal/action, and a confirmable review cannot lack original-worker proof. +Results must match Session, request ID and original journal; confirmation must +match its full request content. Missing legacy evidence remains readable without +confirmation authority. Text shows image kinds, hashes, modes, ownership and native +identity, historical exclusions/created parents and conflicts. Runtime state is +reported separately from observed journal acknowledgement. Confirmation exits 1 +unless runtime settlement is acknowledged; explicit reconciliation additionally +requires an observed native acknowledgement. Read-only lookup may successfully +report a pending/unknown state. Repeated confirmation remains server-side lookup, +including after later owner edits; no client automatically chooses another ID. + +Error hints retain the exact request with shell quoting and an end-of-options +marker, or JSON-escaped identities when controls prevent safe literal display. +The CLI rejects exact `.`/`..` request IDs because URL clients normalize those path +segments; this is documented rather than silently looking up a different route. +Other spaces, slashes and URL punctuation are percent-encoded with their original +identity retained. Source paths and displayed request IDs use escaped quoted +literals so newline/control characters cannot collapse distinct names or affect +the terminal. The operator guide and generated command reference are updated. + +Real service-kill tests now execute typed CLI listing/review/confirmation for both +keep and rollback after a proven partial namespace replacement. They verify stale +fingerprint and foreign Session/request refusal, unchanged transcript/inference +count, original/resolution native receipts, later owner edits, repeated +confirmation and request lookup after another service restart. Explicit typed +reconciliation restores missing journal acknowledgement metadata while retaining a +later source symlink and its target bytes. The final fixture additionally includes +a real newline-containing source filename and checks its unambiguous text display. +API preparation tests remain separate; they are not typed preparation-client proof. + +The code-review skill was applied to command admission, scope correlation, +transport ambiguity/retry identity, terminal escaping, exact URL segments, response +state/acknowledgement semantics and all changed tests/docs. Review corrected the +initial lossy path display and retained unavailable legacy/native proof explicitly. +No unresolved critical/major finding remains for this restore-journal CLI scope. +The cognitive-complexity skill was consulted; its analyzer does not support Rust. +Native Clippy reported the expanded integration fixture at 21 against 15. Extracted +review and foreign-request assertions keep responsibilities clear and satisfy the +native threshold without allowances. An initial sibling-helper visibility error +was also corrected before verification. + +This advances M7 without accepting it: preparation/TUI recovery, actual managed +Git source/runtime integration, partial-constructor recovery, reference-aware +cleanup/cap/retention and full native/platform/client acceptance remain required. +No parent task or issue closes, and no PR/push or source-bound CI is claimed. + +Local macOS arm64 verification: the full serial real CLI branch/recovery suite +passed all 17 cases in 201.36 seconds. After the display correction and fixture +helper extraction, the focused process-kill case passed both keep and rollback +in 51.67 seconds, including the real newline filename, punctuation-bearing request +IDs, foreign/stale refusal, restart lookup and metadata-only acknowledgement. +Final-source CLI unit tests passed all 84 cases, including argument validation, +response scope, escaped labels/hints and generated-reference consistency. Final +all-target locked CLI Clippy passed with Rust warnings denied. All Cargo builds, +checks, tests and Clippy used CARGO_INCREMENTAL=0 and --locked. + +Final format/diff, generated SDK consistency and pinned strict OpenSpec checks +passed (117 items). Spec lint reported 0 errors/25 warnings, with no unchanged-main +origin attribution. Native C deprecation warnings for sandbox_init/ +sandbox_free_error and the CLI oversized unwind-table linker warning were +observed without origin attribution. No separate full snapshot/workspace suite, +SDK runtime suite, TUI/App acceptance, native Linux/Windows execution or +source-bound CI ran for this command-only continuation. Managed Git source +availability remains disabled and all broader completion gates remain open. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 8f1101018..53afbfd09 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -734,3 +734,17 @@ Unchanged generation SHALL preserve file observations and checkpoint identity; publication SHALL use descriptor-relative replacement without following output links. This does not prove user-index or large-untracked eligibility changes; those and full configuration/filter compatibility still gate managed Git admission. + +### Typed recovery CLI dispatch + +`cyber sessions recovery ` SHALL list retained journals without inference. +Providing ` --action keep|rollback` SHALL review only; `--confirm` SHALL +require `--request-id` and the exact reviewed fingerprint. The parser SHALL refuse +incomplete/invalid confirmation combinations before service startup. The separate +`cyber sessions recovery-request ` SHALL perform lookup only; +`--acknowledge` SHALL explicitly reconcile that committed request's native metadata +through the existing route. Errors after confirmation SHALL retain a lookup hint +for the exact request. Text and JSON SHALL distinguish filesystem completion, +durable runtime state and observed native acknowledgement, including unavailable +historical evidence, conflicts and retained created parents. These clients SHALL +NOT manufacture origin proof or bypass unavailable backend/runtime admission. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index d9466acf5..87572b390 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -352,3 +352,9 @@ - **THEN** native planning SHALL refuse before new journal/source effects, including for linked worktrees and scoped Locations - **AND** bounded recipe validation SHALL derive source locators from trusted context, preserve unchanged generation observations, and publish output without following existing file or parent links - **AND** this proof SHALL NOT imply user-index/large-untracked eligibility or activate managed Git admission + +#### Scenario: Typed recovery commands preserve reviewed ownership and retry identity +- **WHEN** the owner lists journals, reviews a selected keep/rollback action and confirms through `sessions recovery` +- **THEN** confirmation SHALL require the exact fingerprint and an explicit stable request ID, and incomplete or malformed combinations SHALL fail before service startup +- **AND** `sessions recovery-request` SHALL look up the exact retained request without replay; only explicit `--acknowledge` SHALL invoke metadata reconciliation +- **AND** text/JSON SHALL distinguish current images, historical evidence availability, conflicts, durable state and observed native acknowledgement; stale/foreign responses or requests SHALL NOT authorize another Session/journal operation diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 2fb812033..085fe4bec 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -460,3 +460,15 @@ configuration/filter compatibility or actual runtime ownership. Managed Git sour admission, M2–M8 and parent tasks remain open. The next client milestone is typed recovery review/confirm/lookup/acknowledgement on the implemented non-Git API, with real restart/retry acceptance; it does not waive remaining Git/GC/platform gates. + +Typed recovery CLI continuation: `sessions recovery` lists retained journals, +reviews an exact keep/rollback selection, and requires both fingerprint and stable +request ID for confirmation. `sessions recovery-request` performs lookup only; +explicit `--acknowledge` invokes the existing metadata-only reconciliation route. +Responses are typed and checked against the selected Session/journal/request; +missing legacy evidence and unobserved native acknowledgement remain explicit. +CLI reference and operator documentation are updated. Real-service kill/restart, +stale/foreign request, repeat lookup and later owner-edit coverage now exercise +these typed commands. This advances the restore-journal client portion of M7; +preparation/TUI clients, actual managed Git integration, safe cleanup and full +native/platform/client acceptance remain open. No parent task is closed. From 0dfb4991d8561d36571a4dc3bb1f8503a03ca812 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 08:30:17 -0400 Subject: [PATCH 38/87] Add terminal snapshot recovery and request discovery --- crates/cyber-cli/tests/branches_cli.rs | 18 + crates/cyber-server/src/http/openapi.rs | 13 +- crates/cyber-server/src/http/sessions.rs | 20 + .../src/runtime/snapshot_recovery.rs | 56 ++- .../src/runtime/snapshot_work_tests.rs | 18 + crates/cyber-tui/src/lib.rs | 1 + crates/cyber-tui/src/recovery.rs | 341 +++++++++++++ crates/cyber-tui/src/recovery_client.rs | 103 ++++ crates/cyber-tui/src/recovery_tests.rs | 451 ++++++++++++++++++ crates/cyber-tui/src/recovery_view.rs | 200 ++++++++ crates/cyber-tui/src/rewind.rs | 18 +- crates/cyber-tui/src/rewind_client.rs | 3 + crates/cyber-tui/src/view/rewind.rs | 8 +- docs/rewind.md | 36 ++ .../implementation.md | 89 ++++ .../managed-recovery.md | 13 + .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 17 + sdk/openapi.json | 63 +++ sdk/typescript/src/generated/operations.ts | 23 + sdk/typescript/src/generated/types.ts | 4 + sdk/typescript/test/rewind.test.ts | 12 + 22 files changed, 1504 insertions(+), 10 deletions(-) create mode 100644 crates/cyber-tui/src/recovery.rs create mode 100644 crates/cyber-tui/src/recovery_client.rs create mode 100644 crates/cyber-tui/src/recovery_tests.rs create mode 100644 crates/cyber-tui/src/recovery_view.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index abcfabca8..bb0dff702 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1026,6 +1026,15 @@ fn assert_foreign_recovery_request_refused(env: &Env, request: &Value) { "{\"title\":\"foreign recovery request\"}", ])); let foreign_id = foreign["data"]["id"].as_str().unwrap(); + let discovered = json(&env.cyber(&[ + "api", + "GET", + &format!( + "/sessions/{foreign_id}/rewind/recovery/requests?journal_id={}", + request["journal_id"].as_str().unwrap() + ), + ])); + assert_eq!(discovered["data"], json!([])); assert!( !env.cyber(&[ "sessions", @@ -1237,6 +1246,15 @@ fn assert_recovery_lookup_does_not_replay( "changed request content reused an existing owner" ); kill_registered_service(env); + let discovered = json(&env.cyber(&[ + "api", + "GET", + &format!( + "/sessions/{id}/rewind/recovery/requests?journal_id={}", + request["journal_id"].as_str().unwrap() + ), + ])); + assert_eq!(discovered["data"], json!([request])); let lookup = json(&env.cyber(&[ "sessions", "recovery-request", diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index 4d84afc4a..ac5f0904c 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -28,6 +28,7 @@ pub struct Op { optional_body: bool, response: Option, query: Vec<&'static str>, + required_query: Vec<&'static str>, located: bool, stream: bool, binary: bool, @@ -49,6 +50,7 @@ fn op(method: &'static str, path: &'static str, id: &'static str, summary: &'sta optional_body: false, response: None, query: Vec::new(), + required_query: Vec::new(), located: false, stream: false, binary: false, @@ -98,6 +100,12 @@ impl Op { self } + fn required_query(mut self, names: &[&'static str]) -> Self { + self.query.extend_from_slice(names); + self.required_query.extend_from_slice(names); + self + } + /// Location-scoped: accepts `location[directory]` and `x-cyber-directory`. fn located(mut self) -> Self { self.located = true; @@ -766,6 +774,9 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { op("post", "/sessions/{sessionID}/rewind/recovery/confirm", "v1.session.snapshotRecoveryConfirm", "Resolve fingerprint-confirmed native snapshot effects with an exact durable request; repeated IDs are lookup-only") .body::(g) .ok::>(g), + op("get", "/sessions/{sessionID}/rewind/recovery/requests", "v1.session.snapshotRecoveryRequests", "Discover up to 4096 retained request identities for this Session creation and exact journal without native access, acknowledgement or replay") + .required_query(&["journal_id"]) + .ok::>>(g), op("get", "/sessions/{sessionID}/rewind/recovery/requests/{requestID}", "v1.session.snapshotRecoveryLookup", "Look up a durable recovery request without native access, acknowledgement reconciliation, replay or inference") .ok::>(g), op("post", "/sessions/{sessionID}/rewind/recovery/requests/{requestID}/acknowledge", "v1.session.snapshotRecoveryAcknowledge", "Reconcile journal acknowledgement for the exact committed native pair without replaying source changes or starting inference") @@ -2047,7 +2058,7 @@ fn operation(o: &Op, error: &Schema, envelope: &Schema) -> Value { if o.id == "v1.memory.delete" { parameters.push(json!({ "name": "x-cyber-memory-review", "in": "header", "required": false, "schema": { "type": "string", "pattern": "^[0-9a-f]{64}$" } })); } - parameters.extend(o.query.iter().map(|name| json!({ "name": name, "in": "query", "required": false, "schema": { "type": "string" } }))); + parameters.extend(o.query.iter().map(|name| json!({ "name": name, "in": "query", "required": o.required_query.contains(name), "schema": { "type": "string" } }))); if o.located { parameters.push(json!({ "name": "location[directory]", "in": "query", "required": false, "schema": { "type": "string" } })); parameters.push(json!({ "name": "x-cyber-directory", "in": "header", "required": false, "schema": { "type": "string" } })); diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index 10761caa4..0d1db9b1e 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -354,6 +354,10 @@ pub fn routes() -> Router { &format!("{s}/rewind/recovery/confirm"), post(snapshot_recovery_confirm), ) + .route( + &format!("{s}/rewind/recovery/requests"), + get(snapshot_recovery_requests), + ) .route( &format!("{s}/rewind/recovery/requests/{{request_id}}"), get(snapshot_recovery_lookup), @@ -1265,6 +1269,22 @@ async fn snapshot_recovery_confirm( .await?, })) } +#[derive(Deserialize)] +struct RecoveryRequestsQuery { + journal_id: String, +} +async fn snapshot_recovery_requests( + State(state): State, + Path(id): Path, + Query(query): Query, +) -> Result>>> { + Ok(Json(Data { + data: state + .runtime + .snapshot_recovery_requests(&id, &query.journal_id) + .await?, + })) +} async fn snapshot_recovery_lookup( State(state): State, Path((id, request_id)): Path<(String, String)>, diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index e98225a8e..df2d923c9 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -120,6 +120,48 @@ impl Runtime { Ok(result) } + /// Read-only discovery of retained request identities; never native authority. + pub async fn snapshot_recovery_requests( + &self, + session: &str, + journal: &str, + ) -> Result, RuntimeError> { + if !valid_journal(journal) { + return Err(RuntimeError::Invalid("Invalid recovery journal".into())); + } + self.session_info(session).await?; + let created = self + .inner + .store + .read_events(session, -1, 1)? + .events + .first() + .ok_or_else(|| RuntimeError::Conflict("Recovery Session creation is missing".into()))? + .id + .clone(); + let session = session.to_owned(); + let journal = journal.to_owned(); + self.inner.store.read(move |db| { + let mut query = db.prepare("SELECT aggregate_id,json_extract(data,'$.snapshot_request.confirmation') FROM event e WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.binary_document') IS NULL AND json_extract(data,'$.snapshot_request.original.session_id')=?1 AND json_extract(data,'$.snapshot_request.original.session_created_id')=?2 AND json_extract(data,'$.snapshot_request.original.binding.journal_id')=?3 AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) ORDER BY e.id DESC LIMIT 4097")?; + let rows = query.query_map(rusqlite::params![session, created, journal], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))?; + let mut requests = Vec::new(); + for row in rows { + let (owner, data) = row?; + let request: super::SnapshotRecoveryConfirmation = serde_json::from_str(&data) + .map_err(|error| cyber_store::StoreError::InvalidAppend(error.to_string()))?; + validate_request(&request).map_err(|error| cyber_store::StoreError::InvalidAppend(error.to_string()))?; + if request.journal_id != journal || owner != request_owner(&created, &request.request_id) { + return Err(cyber_store::StoreError::InvalidAppend("Recovery discovery owner mismatch".into())); + } + requests.push(request); + if requests.len() > 4096 { + return Err(cyber_store::StoreError::InvalidAppend("Recovery discovery limit exceeded".into())); + } + } + Ok(requests) + }).map_err(RuntimeError::from) + } + /// Durable request lookup never opens a native namespace or starts inference. pub async fn snapshot_recovery_lookup( &self, @@ -357,6 +399,13 @@ pub(super) fn validate_request_id(id: &str) -> Result<(), RuntimeError> { } Ok(()) } +fn valid_journal(journal: &str) -> bool { + journal.starts_with("restore_") + && journal.len() <= 256 + && journal + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} pub(super) fn validate_request( request: &super::SnapshotRecoveryConfirmation, ) -> Result<(), RuntimeError> { @@ -366,12 +415,7 @@ pub(super) fn validate_request( .confirmation .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) - || !request.journal_id.starts_with("restore_") - || request.journal_id.len() > 256 - || !request - .journal_id - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + || !valid_journal(&request.journal_id) { return Err(RuntimeError::Invalid( "Invalid recovery journal or fingerprint".into(), diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 1505876b6..ed4401fc1 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -1253,6 +1253,24 @@ async fn cancelling_an_unstarted_resolution_keeps_its_original_failed_restore_fe ) .await .unwrap(); + let discovered = runtime + .snapshot_recovery_requests(&session, &original.binding.journal_id) + .await + .unwrap(); + assert_eq!(discovered, vec![request.confirmation.clone()]); + assert!( + runtime + .snapshot_recovery_requests(&session, "restore_other") + .await + .unwrap() + .is_empty() + ); + assert!( + runtime + .snapshot_recovery_requests(&session, "../restore_other") + .await + .is_err() + ); let lookup = runtime .snapshot_recovery_lookup(&session, &request.confirmation.request_id) .await diff --git a/crates/cyber-tui/src/lib.rs b/crates/cyber-tui/src/lib.rs index 241e75af6..b8d6ac314 100644 --- a/crates/cyber-tui/src/lib.rs +++ b/crates/cyber-tui/src/lib.rs @@ -38,6 +38,7 @@ mod plugins; mod prefs; mod progress; mod recap; +mod recovery; mod rename; mod review; mod rewind; diff --git a/crates/cyber-tui/src/recovery.rs b/crates/cyber-tui/src/recovery.rs new file mode 100644 index 000000000..46ca66146 --- /dev/null +++ b/crates/cyber-tui/src/recovery.rs @@ -0,0 +1,341 @@ +//! Reviewed recovery never edits the composer or infers native acknowledgement. +use crate::{ + app::{Action, App, Overlay}, + rewind, +}; +use crossterm::event::KeyCode; +use cyber_server::runtime::{ + SnapshotRecovery, SnapshotRecoveryAction as RecoveryAction, + SnapshotRecoveryConfirmation as Confirmation, SnapshotRecoveryResult as Outcome, + SnapshotRecoveryReview as Review, SnapshotRecoverySelection as Selection, + SnapshotRecoveryState, +}; +#[path = "recovery_client.rs"] +mod client; +pub use client::perform; +#[path = "recovery_view.rs"] +mod view; +pub use view::lines; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Phase { + #[default] + Journals, + Requests, + Review, + Outcome, +} +#[derive(Debug, Default)] +pub struct View { + pub phase: Phase, + pub busy: bool, + pub discovered: bool, + pub journals: Vec, + pub requests: Vec, + pub selected: usize, + pub journal: Option, + pub review: Option, + pub request: Option, + pub outcome: Option, +} +#[derive(Debug, Clone, PartialEq)] +pub enum Command { + List, + Discover(String), + Review(Selection), + Confirm(Confirmation), + Lookup(Confirmation), + Acknowledge(Confirmation), +} +#[derive(Debug, Clone)] +pub enum Reply { + Journals(Vec), + Requests(Vec), + Review(Box), + Outcome(Box), +} + +impl App { + pub(crate) fn open_snapshot_recovery(&mut self) -> Vec { + self.overlay = Overlay::Rewind; + self.rewind.preview = None; + self.rewind.recovery = Default::default(); + self.rewind.scroll = 0; + self.recovery_request(Command::List) + } + fn recovery_request(&mut self, command: Command) -> Vec { + match &command { + Command::Discover(_) => self.rewind.recovery.discovered = false, + Command::Review(_) => self.rewind.recovery.review = None, + Command::Lookup(_) | Command::Acknowledge(_) => self.rewind.recovery.outcome = None, + _ => {} + } + let actions = self.rewind_request(rewind::Command::Recovery(command)); + self.rewind.phase = rewind::Phase::Recovery; + self.rewind.recovery.busy = true; + actions + } + pub(crate) fn recovery_key(&mut self, key: KeyCode) -> Vec { + if key == KeyCode::Esc { + self.rewind.invalidate(); + self.overlay = Overlay::None; + return vec![]; + } + if key == KeyCode::PageUp { + self.rewind.scroll = self.rewind.scroll.saturating_sub(10); + return vec![]; + } + if key == KeyCode::PageDown { + self.rewind.scroll = self.rewind.scroll.saturating_add(10); + return vec![]; + } + if self.rewind.recovery.busy { + return vec![]; + } + match self.rewind.recovery.phase { + Phase::Journals => self.recovery_journal_key(key), + Phase::Requests => self.recovery_requests_key(key), + Phase::Review => self.recovery_review_key(key), + Phase::Outcome => self.recovery_outcome_key(key), + } + } + fn recovery_select(&mut self, key: KeyCode, length: usize) { + match key { + KeyCode::Up => { + self.rewind.recovery.selected = self.rewind.recovery.selected.saturating_sub(1) + } + KeyCode::Down => { + self.rewind.recovery.selected = + (self.rewind.recovery.selected + 1).min(length.saturating_sub(1)) + } + _ => {} + } + } + fn recovery_journal_key(&mut self, key: KeyCode) -> Vec { + self.recovery_select(key, self.rewind.recovery.journals.len()); + match key { + KeyCode::Char('r') => self.recovery_request(Command::List), + KeyCode::Enter => { + let Some(journal) = self + .rewind + .recovery + .journals + .get(self.rewind.recovery.selected) + else { + return vec![]; + }; + self.recovery_request(Command::Discover(journal.id.clone())) + } + _ => vec![], + } + } + fn recovery_requests_key(&mut self, key: KeyCode) -> Vec { + self.recovery_select(key, self.rewind.recovery.requests.len()); + match key { + KeyCode::Enter => { + let Some(request) = self + .rewind + .recovery + .requests + .get(self.rewind.recovery.selected) + .cloned() + else { + return vec![]; + }; + self.rewind.recovery.request = Some(request.clone()); + self.recovery_request(Command::Lookup(request)) + } + KeyCode::Char('k' | 'b') + if self.rewind.recovery.requests.is_empty() && self.rewind.recovery.discovered => + { + self.recovery_review(key) + } + KeyCode::Char('r') => { + let Some(journal) = self.rewind.recovery.journal.clone() else { + return vec![]; + }; + self.recovery_request(Command::Discover(journal)) + } + _ => vec![], + } + } + fn recovery_review(&mut self, key: KeyCode) -> Vec { + let Some(journal_id) = self.rewind.recovery.journal.clone() else { + return vec![]; + }; + self.rewind.recovery.review = None; + self.rewind.scroll = 0; + self.recovery_request(Command::Review(Selection { + journal_id, + action: if key == KeyCode::Char('k') { + RecoveryAction::Keep + } else { + RecoveryAction::Rollback + }, + })) + } + fn recovery_review_key(&mut self, key: KeyCode) -> Vec { + match key { + KeyCode::Up => self.rewind.scroll = self.rewind.scroll.saturating_sub(1), + KeyCode::Down => self.rewind.scroll = self.rewind.scroll.saturating_add(1), + KeyCode::Enter if !self.session.running => { + let Some(review) = self.rewind.recovery.review.clone() else { + return vec![]; + }; + let Some(confirmation) = review.confirmation else { + return vec![]; + }; + let request = Confirmation { + journal_id: review.journal_id, + action: review.action, + confirmation, + request_id: cyber_core::ids::new_id("recovery"), + }; + self.rewind.recovery.request = Some(request.clone()); + self.rewind.recovery.outcome = None; + self.rewind.recovery.review = None; + self.rewind.recovery.phase = Phase::Outcome; + return self.recovery_request(Command::Confirm(request)); + } + KeyCode::Char('r') => { + let Some(review) = self.rewind.recovery.review.clone() else { + return vec![]; + }; + return self.recovery_request(Command::Review(Selection { + journal_id: review.journal_id, + action: review.action, + })); + } + _ => {} + } + vec![] + } + fn recovery_outcome_key(&mut self, key: KeyCode) -> Vec { + let Some(request) = self.rewind.recovery.request.clone() else { + return vec![]; + }; + match key { + KeyCode::Char('r') => self.recovery_request(Command::Lookup(request)), + KeyCode::Char('a') + if self + .rewind + .recovery + .outcome + .as_ref() + .is_some_and(|outcome| { + outcome.state == SnapshotRecoveryState::Acknowledged + }) => + { + self.recovery_request(Command::Acknowledge(request)) + } + KeyCode::Char('k' | 'b') + if self + .rewind + .recovery + .outcome + .as_ref() + .is_some_and(|outcome| may_review_again(&outcome.state)) => + { + self.recovery_review(key) + } + _ => vec![], + } + } +} + +fn may_review_again(state: &SnapshotRecoveryState) -> bool { + matches!( + state, + SnapshotRecoveryState::CancelledBeforeOrigin + | SnapshotRecoveryState::CancelledBeforeEffects + | SnapshotRecoveryState::ResolvedByAnotherRequest + ) +} + +/// Called only after rewind's Session/Location/generation/overlay guard. +pub(crate) fn apply( + app: &mut App, + request: &rewind::Request, + result: Result, +) -> Vec { + app.rewind.recovery.busy = false; + match result { + Err(error) => { + app.rewind.recovery.review = None; + app.rewind.recovery.outcome = None; + app.rewind.error = Some(error); + } + Ok(rewind::Reply::Recovery(reply)) => { + if let Err(error) = accept(&mut app.rewind.recovery, request, *reply) { + app.rewind.recovery.review = None; + app.rewind.recovery.outcome = None; + app.rewind.error = Some(error); + } + } + _ => app.rewind.error = Some("Unexpected recovery response; refresh without replay".into()), + } + vec![] +} +fn accept(view: &mut View, request: &rewind::Request, reply: Reply) -> Result<(), String> { + let rewind::Command::Recovery(command) = &request.command else { + return Err("Foreign recovery command".into()); + }; + match (command, reply) { + (Command::List, Reply::Journals(journals)) => { + view.journals = journals; + view.phase = Phase::Journals; + view.selected = 0; + } + (Command::Discover(journal), Reply::Requests(requests)) + if requests + .iter() + .all(|request| &request.journal_id == journal) => + { + view.requests = requests; + view.discovered = true; + view.journal = Some(journal.clone()); + view.phase = Phase::Requests; + view.selected = 0; + } + (Command::Review(selection), Reply::Review(review)) + if valid_review(&review, request, selection) => + { + view.review = Some(*review); + view.phase = Phase::Review; + } + ( + Command::Confirm(expected) | Command::Lookup(expected) | Command::Acknowledge(expected), + Reply::Outcome(outcome), + ) if valid_outcome(&outcome, request, expected) => { + view.request = Some(expected.clone()); + view.outcome = Some(*outcome); + view.phase = Phase::Outcome; + } + _ => { + return Err( + "Recovery reply does not match the selected owner; refresh without replay".into(), + ); + } + } + Ok(()) +} + +fn valid_review(review: &Review, request: &rewind::Request, selection: &Selection) -> bool { + review.journal_id == selection.journal_id + && review.action == selection.action + && (review.confirmation.is_none() || review.origin.is_some()) + && review.origin.as_ref().is_none_or(|origin| { + origin.session_id == request.session + && origin.binding.directory == std::path::Path::new(&request.directory) + && origin.binding.journal_id == selection.journal_id + }) +} +fn valid_outcome(outcome: &Outcome, request: &rewind::Request, expected: &Confirmation) -> bool { + outcome.request == *expected + && outcome.original.session_id == request.session + && outcome.original.binding.directory == std::path::Path::new(&request.directory) + && outcome.original.binding.journal_id == expected.journal_id +} +#[cfg(test)] +#[path = "recovery_tests.rs"] +mod tests; diff --git a/crates/cyber-tui/src/recovery_client.rs b/crates/cyber-tui/src/recovery_client.rs new file mode 100644 index 000000000..3bed3b720 --- /dev/null +++ b/crates/cyber-tui/src/recovery_client.rs @@ -0,0 +1,103 @@ +use super::{Command, Reply}; +use crate::{model::Session, rewind::Request}; +use cyber_client::Client; +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; + +fn segment(value: &str) -> String { + value + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || b"-_.~".contains(&byte) { + (byte as char).to_string() + } else { + format!("%{byte:02X}") + } + }) + .collect() +} +fn data(reply: Value) -> Result { + serde_json::from_value(reply["data"].clone()) + .map_err(|error| format!("Invalid recovery reply: {error}")) +} +pub async fn perform( + client: &Client, + request: &Request, + command: &Command, +) -> Result { + let session = Session::parse( + &client + .get(&format!("/sessions/{}", segment(&request.session))) + .await + .map_err(|error| error.to_string())?["data"], + ); + if session.id != request.session || session.directory != request.directory { + return Err("Recovery Session/Location changed; refresh without replay".into()); + } + let base = format!("/sessions/{}/rewind/recovery", segment(&request.session)); + match command { + Command::List => { + let journals: Vec = + data(client.get(&base).await.map_err(|error| error.to_string())?)?; + if journals.len() > 4096 { + return Err("Recovery journal limit exceeded".into()); + } + Ok(Reply::Journals(journals)) + } + Command::Discover(journal) => { + let requests: Vec = data( + client + .get(&format!("{base}/requests?journal_id={}", segment(journal))) + .await + .map_err(|error| error.to_string())?, + )?; + if requests.len() > 4096 { + return Err("Recovery request limit exceeded".into()); + } + Ok(Reply::Requests(requests)) + } + Command::Review(selection) => Ok(Reply::Review(Box::new(data( + client + .post(&format!("{base}/review"), json!(selection)) + .await + .map_err(|error| error.to_string())?, + )?))), + Command::Confirm(confirmation) => Ok(Reply::Outcome(Box::new(data( + client + .post(&format!("{base}/confirm"), json!(confirmation)) + .await + .map_err(|error| { + format!( + "{error}. Retain request {:?}; R looks it up without replay", + confirmation.request_id + ) + })?, + )?))), + Command::Lookup(confirmation) => { + if matches!(confirmation.request_id.as_str(), "." | "..") { + return Err("This request ID is not addressable through the path lookup API; retained evidence is unchanged".into()); + } + Ok(Reply::Outcome(Box::new(data( + client + .get(&format!( + "{base}/requests/{}", + segment(&confirmation.request_id) + )) + .await + .map_err(|error| error.to_string())?, + )?))) + } + Command::Acknowledge(confirmation) => Ok(Reply::Outcome(Box::new(data( + client + .post( + &format!( + "{base}/requests/{}/acknowledge", + segment(&confirmation.request_id) + ), + json!({}), + ) + .await + .map_err(|error| error.to_string())?, + )?))), + } +} diff --git a/crates/cyber-tui/src/recovery_tests.rs b/crates/cyber-tui/src/recovery_tests.rs new file mode 100644 index 000000000..c6075cdad --- /dev/null +++ b/crates/cyber-tui/src/recovery_tests.rs @@ -0,0 +1,451 @@ +use super::*; +use crate::{model::Session, perform::Msg}; +use serde_json::json; + +fn request(actions: Vec) -> rewind::Request { + let [Action::Rewind(request)] = actions.as_slice() else { + panic!("one owner request") + }; + request.clone() +} +fn deliver(app: &mut App, request: rewind::Request, reply: Reply) { + rewind::apply( + app, + &mut Default::default(), + request, + Ok(rewind::Reply::Recovery(Box::new(reply))), + ); +} +fn origin() -> cyber_server::runtime::SnapshotOrigin { + serde_json::from_value(json!({"version":1,"session_id":"ses_one","session_created_id":"created","activity_id":"act_one","worktree_id":null,"binding":{"directory":"/project","root_identity":"root","namespace_identity":"namespace","journal_id":"restore_one","journal_identity":"journal"}})).unwrap() +} +fn review() -> Review { + Review { + journal_id: "restore_one".into(), + action: RecoveryAction::Rollback, + origin: Some(origin()), + filesystem_complete: false, + acknowledged: false, + confirmation: Some("a".repeat(64)), + paths: vec![], + exclusions: None, + recorded_created_parents: None, + conflicts: vec![], + } +} +fn app() -> App { + let mut app = App::new( + Session { + id: "ses_one".into(), + directory: "/project".into(), + ..Default::default() + }, + vec![], + "cyber", + ); + app.composer.set_text("owner draft"); + app +} +#[test] +fn recovery_drops_stale_generations_and_foreign_sessions_without_changing_drafts() { + let mut app = app(); + let old = request(app.open_snapshot_recovery()); + let current = request(app.open_snapshot_recovery()); + deliver(&mut app, old, Reply::Journals(vec![])); + assert!(app.rewind.recovery.busy); + deliver(&mut app, current.clone(), Reply::Journals(vec![])); + assert!(!app.rewind.recovery.busy); + app.set_session(Session { + id: "ses_foreign".into(), + directory: "/foreign".into(), + ..Default::default() + }); + app.composer.set_text("foreign draft"); + deliver(&mut app, current, Reply::Journals(vec![])); + assert_eq!(app.composer.text(), "foreign draft"); + assert_ne!(app.rewind.phase, rewind::Phase::Recovery); +} +#[test] +fn ambiguous_confirmation_retains_exact_lookup_and_never_reuses_enter_to_mutate() { + let mut app = app(); + let initial = request(app.open_snapshot_recovery()); + deliver(&mut app, initial, Reply::Journals(vec![])); + app.rewind.recovery.phase = Phase::Requests; + app.rewind.recovery.discovered = true; + app.rewind.recovery.journal = Some("restore_one".into()); + let selection = request(app.recovery_key(KeyCode::Char('b'))); + deliver(&mut app, selection, Reply::Review(Box::new(review()))); + let confirmation = request(app.recovery_key(KeyCode::Enter)); + let rewind::Command::Recovery(Command::Confirm(body)) = confirmation.command.clone() else { + panic!("explicit confirmation") + }; + assert!(app.rewind.recovery.review.is_none()); + assert!(app.recovery_key(KeyCode::Enter).is_empty()); + rewind::apply( + &mut app, + &mut Default::default(), + confirmation, + Err("reply lost".into()), + ); + assert!(app.recovery_key(KeyCode::Enter).is_empty()); + let lookup = request(app.recovery_key(KeyCode::Char('r'))); + assert_eq!( + lookup.command, + rewind::Command::Recovery(Command::Lookup(body.clone())) + ); + let mut outcome = Outcome { + request: body, + original: origin(), + resolution: None, + state: SnapshotRecoveryState::Acknowledged, + receipt: None, + journal_acknowledged: None, + }; + outcome.original.session_id = "ses_foreign".into(); + deliver(&mut app, lookup, Reply::Outcome(Box::new(outcome))); + assert!(app.rewind.recovery.outcome.is_none()); + assert!(app.recovery_key(KeyCode::Char('a')).is_empty()); + assert_eq!(app.composer.text(), "owner draft"); +} + +#[test] +fn failed_discovery_does_not_license_a_fresh_attempt() { + let mut app = app(); + let load = request(app.open_snapshot_recovery()); + deliver( + &mut app, + load, + Reply::Journals(vec![SnapshotRecovery { + id: "restore_one".into(), + filesystem_complete: false, + paths: vec![], + applied: vec![], + }]), + ); + let discovery = request(app.recovery_key(KeyCode::Enter)); + deliver(&mut app, discovery, Reply::Requests(vec![])); + let discovery = request(app.recovery_key(KeyCode::Char('r'))); + rewind::apply( + &mut app, + &mut Default::default(), + discovery, + Err("discovery unavailable".into()), + ); + assert!(app.recovery_key(KeyCode::Char('k')).is_empty()); + assert!(app.recovery_key(KeyCode::Char('b')).is_empty()); + assert_eq!(app.composer.text(), "owner draft"); +} + +#[test] +fn recovery_review_escapes_paths_and_legacy_evidence_has_no_confirm_action() { + let mut app = app(); + let _ = app.open_snapshot_recovery(); + let mut review = review(); + review + .paths + .push(cyber_server::runtime::SnapshotRecoveryPath { + path: "one\ntwo\x1b[31m".into(), + state: "unapplied".into(), + current: None, + recorded_before: None, + recorded_after: None, + retained: None, + proposed: None, + changes: false, + }); + review.origin = None; + review.confirmation = None; + app.rewind.recovery.busy = false; + app.rewind.recovery.phase = Phase::Review; + app.rewind.recovery.review = Some(review); + let shown = lines(&app) + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n"); + assert!(shown.contains("one\\ntwo")); + assert!(!shown.contains('\x1b')); + assert!(shown.contains("Confirmation unavailable")); + assert!(app.recovery_key(KeyCode::Enter).is_empty()); + assert_eq!(app.composer.text(), "owner draft"); +} + +#[test] +fn only_positively_settled_attempts_allow_a_new_review_without_replay() { + for state in [ + SnapshotRecoveryState::Pending, + SnapshotRecoveryState::Unknown, + SnapshotRecoveryState::CancelledBeforeOrigin, + SnapshotRecoveryState::CancelledBeforeEffects, + SnapshotRecoveryState::ResolvedByAnotherRequest, + ] { + let mut app = app(); + let _ = app.open_snapshot_recovery(); + let body = Confirmation { + journal_id: "restore_one".into(), + action: RecoveryAction::Keep, + confirmation: "a".repeat(64), + request_id: "retained".into(), + }; + app.rewind.recovery.busy = false; + app.rewind.recovery.phase = Phase::Outcome; + app.rewind.recovery.journal = Some(body.journal_id.clone()); + app.rewind.recovery.request = Some(body.clone()); + app.rewind.recovery.outcome = Some(Outcome { + request: body, + original: origin(), + resolution: None, + state, + receipt: None, + journal_acknowledged: None, + }); + let actions = app.recovery_key(KeyCode::Char('b')); + if matches!( + state, + SnapshotRecoveryState::Pending | SnapshotRecoveryState::Unknown + ) { + assert!(actions.is_empty()); + } else { + assert!(matches!( + request(actions).command, + rewind::Command::Recovery(Command::Review(_)) + )); + } + assert_eq!(app.composer.text(), "owner draft"); + } +} + +async fn run(server: &crate::screens::Server, app: &mut App, mut actions: Vec) { + while let Some(action) = actions.pop() { + let message = + crate::perform::perform_owned(&server.client, &app.session, action, None).await; + actions.extend(crate::runner::apply(app, message, &mut Default::default())); + } +} +#[tokio::test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn terminal_recovery_handles_real_partial_interruption_and_discovers_a_lost_reply_after_reopen() + { + for action in [KeyCode::Char('k'), KeyCode::Char('b')] { + native_recovery(action).await; + } +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn native_recovery(action: KeyCode) { + let mut fixture = native_fixture().await; + interrupt_restore(&mut fixture).await; + let inference = fixture.server.model.requests().len(); + confirm_without_applying_reply(&mut fixture, action).await; + reopen_and_acknowledge(&fixture, inference).await; +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +struct NativeFixture { + server: crate::screens::Server, + client: cyber_client::Client, + directory: std::path::PathBuf, + id: String, + messages: serde_json::Value, + app: App, +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn native_fixture() -> NativeFixture { + let server = crate::screens::Server::start_config(json!({}), json!({}), true).await; + let directory = server.root().join("recovery-work"); + std::fs::create_dir(&directory).unwrap(); + bulk(&directory, "before"); + let client = server.client.at(directory.to_str().unwrap()); + let created = client + .post( + "/sessions", + json!({"model":"test/main","mode":"accept-edits"}), + ) + .await + .unwrap(); + let id = created["data"]["id"].as_str().unwrap().to_owned(); + server.prompt(&id, "before checkpoint").await; + server.wait_answered(&id).await; + bulk(&directory, "after"); + server.prompt(&id, "after checkpoint").await; + server.wait_answered(&id).await; + let messages = client + .get(&format!("/sessions/{id}/messages")) + .await + .unwrap(); + let mut app = server.load(&id).await; + app.composer.set_text("owner draft"); + NativeFixture { + server, + client, + directory, + id, + messages, + app, + } +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn interrupt_restore(fixture: &mut NativeFixture) { + let (server, client, directory, id, app) = ( + &fixture.server, + &fixture.client, + &fixture.directory, + &fixture.id, + &mut fixture.app, + ); + let actions = app.open_rewind(rewind::Intent::Menu); + run(server, app, actions).await; + app.rewind.selected = 0; + app.rewind.target = cyber_server::runtime::RevertTarget::Code; + let actions = app.rewind_key(crossterm::event::KeyEvent::from(KeyCode::Enter)); + run(server, app, actions).await; + assert_eq!(app.rewind.phase, rewind::Phase::Review); + let pending = app + .rewind_key(crossterm::event::KeyEvent::from(KeyCode::Enter)) + .pop() + .unwrap(); + let (owned, session) = ((*client).clone(), app.session.clone()); + let task = tokio::spawn(async move { + crate::perform::perform_owned(&owned, &session, pending, None).await + }); + wait_for_partial(directory, &task).await; + assert!( + server.host.runtime.interrupt_for_owner(id).await.is_err(), + "partial native effects must retain uncertainty" + ); + let reply = task.await.unwrap(); + assert!(matches!(&reply, Ok(Msg::Rewind { result, .. }) if result.is_err())); + crate::runner::apply(app, reply, &mut Default::default()); + assert_eq!(app.composer.text(), "owner draft"); +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn wait_for_partial( + directory: &std::path::Path, + task: &tokio::task::JoinHandle>, +) { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(45); + while std::fs::read_to_string(directory.join("bulk_0000")).unwrap() != "before" { + assert!( + !task.is_finished(), + "restore finished without a partial replacement" + ); + assert!( + tokio::time::Instant::now() < deadline, + "source exchange was not observed" + ); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + assert_eq!( + std::fs::read_to_string(directory.join("bulk_1279")).unwrap(), + "after" + ); +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn confirm_without_applying_reply(fixture: &mut NativeFixture, action: KeyCode) { + let (server, client, app) = (&fixture.server, &fixture.client, &mut fixture.app); + std::fs::write(fixture.directory.join("unrelated-owner"), "owner data").unwrap(); + let actions = app.open_snapshot_recovery(); + run(server, app, actions).await; + let actions = app.recovery_key(KeyCode::Enter); + run(server, app, actions).await; + assert!(app.rewind.recovery.requests.is_empty()); + let actions = app.recovery_key(action); + run(server, app, actions).await; + assert!( + app.rewind + .recovery + .review + .as_ref() + .unwrap() + .confirmation + .is_some() + ); + let shown = crate::screens::render(app, 120, 35, &Default::default()); + assert!(shown.contains("SNAPSHOT RECOVERY") && shown.contains("REVIEW")); + let confirm = app.recovery_key(KeyCode::Enter).pop().unwrap(); + let acknowledgement = crate::perform::perform_owned(client, &app.session, confirm, None) + .await + .unwrap(); + let Msg::Rewind { result, .. } = acknowledgement else { + panic!("recovery response") + }; + assert!(result.is_ok()); // Deliberately discard the successful reply. + let expected = if action == KeyCode::Char('k') { + "before" + } else { + "after" + }; + assert_eq!( + std::fs::read_to_string(fixture.directory.join("bulk_0000")).unwrap(), + expected + ); + assert_eq!( + std::fs::read_to_string(fixture.directory.join("unrelated-owner")).unwrap(), + "owner data" + ); +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +async fn reopen_and_acknowledge(fixture: &NativeFixture, inference: usize) { + let NativeFixture { + server, + client, + directory, + id, + messages, + .. + } = fixture; + let mut fresh = server.load(id).await; + fresh.composer.set_text("new terminal draft"); + let actions = fresh.open_snapshot_recovery(); + run(server, &mut fresh, actions).await; + let actions = fresh.recovery_key(KeyCode::Enter); + run(server, &mut fresh, actions).await; + assert_eq!(fresh.rewind.recovery.requests.len(), 1); + let actions = fresh.recovery_key(KeyCode::Enter); + run(server, &mut fresh, actions).await; + assert_eq!( + fresh.rewind.recovery.outcome.as_ref().unwrap().state, + SnapshotRecoveryState::Acknowledged + ); + assert!( + fresh + .rewind + .recovery + .outcome + .as_ref() + .unwrap() + .journal_acknowledged + .is_none() + ); + std::fs::write(directory.join("bulk_0000"), "later owner edit").unwrap(); + let actions = fresh.recovery_key(KeyCode::Char('a')); + run(server, &mut fresh, actions).await; + assert_eq!( + fresh + .rewind + .recovery + .outcome + .as_ref() + .unwrap() + .journal_acknowledged, + Some(true) + ); + assert_eq!( + std::fs::read_to_string(directory.join("bulk_0000")).unwrap(), + "later owner edit" + ); + assert_eq!(fresh.composer.text(), "new terminal draft"); + assert_eq!( + client + .get(&format!("/sessions/{id}/messages")) + .await + .unwrap(), + *messages + ); + assert_eq!(server.model.requests().len(), inference); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn bulk(directory: &std::path::Path, contents: &str) { + for index in 0..1280 { + std::fs::write(directory.join(format!("bulk_{index:04}")), contents).unwrap(); + } +} diff --git a/crates/cyber-tui/src/recovery_view.rs b/crates/cyber-tui/src/recovery_view.rs new file mode 100644 index 000000000..d9e0223cc --- /dev/null +++ b/crates/cyber-tui/src/recovery_view.rs @@ -0,0 +1,200 @@ +use super::{Phase, View}; +use crate::{app::App, messaging::clean}; +use cyber_server::runtime::{SnapshotRecoveryImage, SnapshotRecoveryState}; +use ratatui::text::Line; + +pub fn lines(app: &App) -> Vec> { + let view = &app.rewind.recovery; + let mut lines = vec![ + Line::from("SNAPSHOT RECOVERY · composer and conversation stay unchanged"), + Line::from("Esc closes · PgUp/PgDn scroll · external effects remain retained"), + ]; + if view.busy { + lines.push(Line::from( + "Waiting for owner server; closing does not cancel an admitted worker.", + )); + } + if let Some(request) = &view.request { + lines.push(Line::from(format!( + "Retained request: {:?}", + request.request_id + ))); + } + match view.phase { + Phase::Journals => journals(view, &mut lines), + Phase::Requests => requests(view, &mut lines), + Phase::Review => review(view, &mut lines), + Phase::Outcome => outcome(view, &mut lines), + } + lines +} +fn journals(view: &View, lines: &mut Vec>) { + lines.push(Line::from( + "↑/↓ select journal · Enter discovers retained requests · R reloads", + )); + if view.journals.is_empty() { + lines.push(Line::from("No retained journals. Pre-effect preparations may still need their preparation API; no fence is cleared.")); + } + for (index, journal) in view + .journals + .iter() + .enumerate() + .skip(view.selected.saturating_sub(4)) + .take(9) + { + lines.push(Line::from(format!( + "{} {} · filesystem complete={} · recorded progress {}/{} (not acknowledgement)", + if index == view.selected { ">" } else { " " }, + clean(&journal.id), + journal.filesystem_complete, + journal.applied.len(), + journal.paths.len() + ))); + } +} +fn requests(view: &View, lines: &mut Vec>) { + lines.push(Line::from(format!( + "Journal: {}", + clean(view.journal.as_deref().unwrap_or("unavailable")) + ))); + if !view.discovered { + lines.push(Line::from( + "Request discovery unavailable; R retries without starting another attempt.", + )); + return; + } + if view.requests.is_empty() { + lines.push(Line::from("No retained request. K reviews keep; B reviews rollback. Neither resumes the failed rewind.")); + } else { + lines.push(Line::from("Retained requests found. ↑/↓ select · Enter looks up · R refreshes discovery. Inspect existing outcomes before another attempt.")); + } + for (index, request) in view + .requests + .iter() + .enumerate() + .skip(view.selected.saturating_sub(4)) + .take(9) + { + lines.push(Line::from(format!( + "{} {:?} · {:?}", + if index == view.selected { ">" } else { " " }, + request.request_id, + request.action + ))); + } +} +fn image(value: Option<&SnapshotRecoveryImage>) -> String { + value.map_or_else( + || "absent".into(), + |image| { + format!( + "{:?} {} · {} bytes · mode {:o} · uid {} gid {} · native {:?}", + image.kind, + clean(&image.hash), + image.bytes, + image.mode, + image.uid, + image.gid, + image.native_identity + ) + }, + ) +} +fn review(view: &View, lines: &mut Vec>) { + let Some(review) = &view.review else { + return; + }; + lines.push(Line::from(format!( + "REVIEW {:?} · {} · filesystem complete={} · acknowledged={}", + review.action, + clean(&review.journal_id), + review.filesystem_complete, + review.acknowledged + ))); + if let Some(origin) = &review.origin { + lines.push(Line::from(format!( + "Original Session {} · worker {} · Location {:?}", + clean(&origin.session_id), + clean(&origin.activity_id), + origin.binding.directory + ))); + } else { + lines.push(Line::from("Original worker identity unavailable")); + } + for path in &review.paths { + lines.push(Line::from(format!( + "{:?} [{}] changes={}", + path.path, + clean(&path.state), + path.changes + ))); + for (label, image_value) in [ + ("current", &path.current), + ("before", &path.recorded_before), + ("after", &path.recorded_after), + ("retained", &path.retained), + ("proposed", &path.proposed), + ] { + lines.push(Line::from(format!( + " {label}: {}", + image(image_value.as_ref()) + ))); + } + } + paths("Historical exclusions", review.exclusions.as_deref(), lines); + paths( + "Protected created parents", + review.recorded_created_parents.as_deref(), + lines, + ); + for conflict in &review.conflicts { + lines.push(Line::from(format!("Conflict: {conflict:?}"))); + } + if let Some(hash) = &review.confirmation { + lines.push(Line::from(format!( + "Fingerprint {} · Enter explicitly confirms · R refreshes review", + clean(hash) + ))); + } else { + lines.push(Line::from("Confirmation unavailable; retain evidence.")); + } +} +fn paths(label: &str, paths: Option<&[String]>, lines: &mut Vec>) { + match paths { + None => lines.push(Line::from(format!( + "{label}: unavailable historical evidence" + ))), + Some([]) => lines.push(Line::from(format!("{label}: none"))), + Some(paths) => lines.extend( + paths + .iter() + .map(|path| Line::from(format!("{label}: {path:?}"))), + ), + } +} +fn outcome(view: &View, lines: &mut Vec>) { + lines.push(Line::from( + "R looks up this exact request; Enter never repeats source work.", + )); + let Some(outcome) = &view.outcome else { + lines.push(Line::from("Outcome not observed. Refresh lookup, or reopen C recovery to discover the reserved request after restart.")); + return; + }; + lines.push(Line::from(format!( + "Runtime state: {:?} · native acknowledgement: {}", + outcome.state, + outcome + .journal_acknowledged + .map_or("not observed by this response", |value| if value { + "observed" + } else { + "not acknowledged" + }) + ))); + match outcome.state { + SnapshotRecoveryState::Acknowledged => lines.push(Line::from("A explicitly reconciles native acknowledgement metadata; source files are never replayed.")), + SnapshotRecoveryState::CancelledBeforeOrigin | SnapshotRecoveryState::CancelledBeforeEffects => lines.push(Line::from("This attempt was cancelled before effects. The original restore is unresolved; K/B requests a fresh reviewed keep/rollback attempt.")), + SnapshotRecoveryState::ResolvedByAnotherRequest => lines.push(Line::from("This attempt was resolved by another owned request. K/B requests a fresh review of the original restore; no source work is replayed.")), + _ => lines.push(Line::from("Unresolved native evidence remains fenced. Preparation cancellation requires its exact pre-effect proof; no force-clear is available.")), + } +} diff --git a/crates/cyber-tui/src/rewind.rs b/crates/cyber-tui/src/rewind.rs index b593f00c4..dc6427280 100644 --- a/crates/cyber-tui/src/rewind.rs +++ b/crates/cyber-tui/src/rewind.rs @@ -31,6 +31,7 @@ pub enum Phase { Review, Busy, Waiting, + Recovery, } #[derive(Debug, Clone)] pub struct Boundary { @@ -45,6 +46,7 @@ struct Draft { #[derive(Debug)] pub struct View { + pub recovery: crate::recovery::View, pub generation: u64, pub phase: Phase, pub intent: Intent, @@ -63,6 +65,7 @@ pub struct View { impl Default for View { fn default() -> Self { Self { + recovery: Default::default(), generation: 0, phase: Phase::Closed, intent: Intent::Menu, @@ -82,6 +85,7 @@ impl Default for View { } impl View { pub fn invalidate(&mut self) { + self.recovery = Default::default(); self.generation = self.generation.wrapping_add(1); self.phase = Phase::Closed; self.preview = None; @@ -97,6 +101,7 @@ impl View { } #[derive(Debug, Clone, PartialEq)] pub enum Command { + Recovery(crate::recovery::Command), Load, Interrupt, Preview(RewindSelection), @@ -112,6 +117,7 @@ pub struct Request { } #[derive(Debug, Clone)] pub enum Reply { + Recovery(Box), Loaded(Box), Interrupted(Box), Preview(Box), @@ -173,7 +179,7 @@ impl App { self.rewind_request(Command::Load) } } - fn rewind_request(&mut self, command: Command) -> Vec { + pub(crate) fn rewind_request(&mut self, command: Command) -> Vec { self.rewind.generation = self.rewind.generation.wrapping_add(1); self.rewind.phase = Phase::Loading; self.rewind.error = None; @@ -200,6 +206,12 @@ impl App { self.rewind_request(Command::Preview(selection)) } pub(crate) fn rewind_key(&mut self, key: KeyEvent) -> Vec { + if self.rewind.phase == Phase::Recovery { + return self.recovery_key(key.code); + } + if key.code == KeyCode::Char('c') && !matches!(self.rewind.phase, Phase::Loading) { + return self.open_snapshot_recovery(); + } if key.code == KeyCode::Esc { self.rewind.generation = self.rewind.generation.wrapping_add(1); self.rewind.preview = None; @@ -377,7 +389,11 @@ pub(crate) fn apply( { return vec![]; } + if matches!(&request.command, Command::Recovery(_)) { + return crate::recovery::apply(app, &request, result); + } match result { + Ok(Reply::Recovery(_)) => vec![], Err(error) => { if matches!(&request.command,Command::Preview(s) if s.target.code()) { app.rewind.code_error = Some(error.clone()); diff --git a/crates/cyber-tui/src/rewind_client.rs b/crates/cyber-tui/src/rewind_client.rs index e3c280bd1..ca1f87d61 100644 --- a/crates/cyber-tui/src/rewind_client.rs +++ b/crates/cyber-tui/src/rewind_client.rs @@ -35,6 +35,9 @@ async fn load(client: &Client, request: &Request) -> Result { async fn run(client: &Client, request: &Request) -> Result { let route = format!("/sessions/{}/rewind", request.session); match &request.command { + Command::Recovery(command) => crate::recovery::perform(client, request, command) + .await + .map(|reply| Reply::Recovery(Box::new(reply))), Command::Load => Ok(Reply::Loaded(Box::new(load(client, request).await?))), Command::Preview(selection) => { let response = client diff --git a/crates/cyber-tui/src/view/rewind.rs b/crates/cyber-tui/src/view/rewind.rs index 37e6f1cd6..801228701 100644 --- a/crates/cyber-tui/src/view/rewind.rs +++ b/crates/cyber-tui/src/view/rewind.rs @@ -47,12 +47,16 @@ pub fn draw(frame: &mut Frame, app: &App, area: Rect) { "Waiting for the owner server… Esc closes. Failed confirmations require fresh status and review.", )); } + Phase::Recovery => lines.extend(crate::recovery::lines(app)), Phase::Review => review(app, &mut lines), _ => choose(app, &mut lines), } if let Some(error) = &view.code_error { lines.push(Line::styled( - format!("Code unavailable/conflicting: {}", clean(error)), + format!( + "Code unavailable/conflicting: {} · C inspects retained recovery", + clean(error) + ), Style::default().fg(app.theme.warning), )); } @@ -76,7 +80,7 @@ fn choose(app: &App, lines: &mut Vec>) { view.target, view.action ))); lines.push(Line::from( - "↑/↓ select · Enter preview · U undo · D redo · R reload · Esc close", + "↑/↓ select · Enter preview · U undo · D redo · C recovery · R reload · Esc close", )); if view.saved_draft(&app.session).is_some() { lines.push(Line::from("P copies the saved original text draft")); diff --git a/docs/rewind.md b/docs/rewind.md index 6d4215deb..81fbbc2b5 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -406,3 +406,39 @@ Linux/macOS hosts. Ordinary managed Git source recovery remains disabled pending its remaining compatibility and runtime acceptance gates. Preparation cancellation is still available through its existing API; typed preparation/TUI recovery, reference-aware reclamation and complete native/platform acceptance remain open. + +## Recovery in the terminal panel + +Open `/rewind` (or idle Esc-Esc) and press **C** to inspect recovery. This entry +also remains available when code rewind reports unavailable or conflicting state. +Select a journal with Up/Down, then Enter to discover its retained requests. + +- If there are no requests, **K** reviews keep and **B** reviews rollback. The + review shows original worker identity, images, historical exclusions, protected + created parents and conflicts. A second Enter confirms its fingerprint. +- If requests already exist, select one and press Enter to look it up. A fresh + terminal uses these server-retained identities after a lost reply; it does not + choose another request ID to replay the operation. +- **R** refreshes the current read/review. After confirmation or an ambiguous + response it performs exact request lookup. Enter never retries source work from + an outcome view. +- An acknowledged runtime result may still have unobserved native acknowledgement. + **A** explicitly reconciles that request's native metadata without reinstalling + source files. A positively cancelled pre-effect attempt, or an attempt resolved by another + owned request, offers K/B for a fresh review of the original restore; + pending/unknown outcomes retain their fences. +- PgUp/PgDn scroll; Esc closes. Closing does not cancel an already admitted native + worker. Reopen recovery to discover its retained request. + +Recovery preserves the composer, attachments and conversation. Replies from an old +Session, Location or UI generation are ignored. Missing legacy proof never enables +confirmation, and failed request discovery cannot license a fresh attempt. +Preparation cancellation still uses the existing preparation API; its typed panel +and the remaining managed Git/cleanup/platform acceptance work are unfinished. + +`GET /api/v1/sessions/{id}/rewind/recovery/requests?journal_id=restore_example` +provides the read-only discovery used by the panel. It requires an exact journal, +filters to the current Session creation, verifies request-owner identities and +returns at most 4096 retained confirmation records. Overflow refuses explicitly +without deleting history. Discovery does not inspect source files, acknowledge +native work or start inference. The SDK exposes `session.snapshotRecoveryRequests`. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 66f915f40..7b56821e4 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2892,3 +2892,92 @@ observed without origin attribution. No separate full snapshot/workspace suite, SDK runtime suite, TUI/App acceptance, native Linux/Windows execution or source-bound CI ran for this command-only continuation. Managed Git source availability remains disabled and all broader completion gates remain open. + +## Terminal restore recovery and retained-request discovery (2026-11-07) + +Rewind C now opens a recovery panel without inference. Journal selection performs +read-only request discovery before permitting a new attempt. K/B reviews an exact +keep/rollback selection; Enter confirms only a confirmable fingerprint and generates +one retained request identity. Confirmation immediately drops its preview. An +ambiguous response retains that identity and R performs lookup, never source +replay. A separately reconciles native metadata for an acknowledged runtime result. +A positively cancelled pre-effect request, or one resolved by another owned +request, can lead to a fresh explicit review of the original restore. Pending and +unknown outcomes cannot license another attempt. No force-clear is introduced. + +The controller uses rewind's existing Session/Location/UI-generation/overlay guard +and never invokes rewind's composer-replacement finish path. It validates full +confirmation identity and original Session/directory/journal on results; reviews +must match their selected action/journal and cannot confirm without original +worker identity. Closing or switching scope invalidates late replies without +cancelling or replaying an admitted worker. Display preserves exact quoted paths +and request IDs with escaped controls, distinguishes absent historical evidence +and separates runtime settlement from observed native acknowledgement. Review and +outcome capabilities are dropped on failed/mismatched replies. A failed refresh of +previously empty discovery also disables fresh-attempt keys until discovery +succeeds, rather than treating cached emptiness as current evidence. + +New authenticated GET recovery/requests with required journal_id discovers up to +4096 retained confirmation identities for the current Session creation and exact +journal. It reads latest native snapshot-worker records, excludes binary document +owners, validates request syntax and recomputes each exact request-owner identity. +Overflow refuses without deleting history; discovery is not acknowledgement or +native origin authority. It does not open native namespaces or inspect source +files. This server-side discovery lets a fresh TUI App recover a lost request +without making an in-memory ID or local preference file its sole retry identity. +OpenAPI, generated TypeScript operations/types, SDK tests and the operator guide +are synchronized. Query metadata now declares the required journal selector. + +Two real production-App/native scenarios create 1280 changed files, start a +reviewed rewind through the TUI action port, observe an actual first replacement +with the last file still untouched, and interrupt the exact live worker. The +runtime correctly retains uncertainty and returns recovery-required; the original +fixture's expectation of successful interruption was corrected. Both keep and +rollback are then reviewed and confirmed through the TUI/public API. Assertions +verify their distinct first-file results, unrelated owner data, the original draft, +no transcript movement and no inference. After discarding the successful recovery +reply, a freshly loaded TUI App discovers the existing request, reads its durable +outcome and explicitly reconciles native acknowledgement while preserving a later +owner edit and new terminal draft. These are native cancellation and fresh-App +cases against a live server, not OS process kills of the TUI or server. The updated +real CLI service-kill case separately verifies discovery after server restart and +empty discovery for a foreign Session. Existing runtime cancellation coverage also +verifies discovery of an exactly owned cancelled request, journal filtering and +invalid selector refusal. + +The code-review skill was applied to bounded/parameterized discovery, Session +creation and native-owner filtering, response correlation, UI generation fencing, +duplicate/ambiguous requests, proof availability, terminal display, independent +native acknowledgement and all changed tests/generated artifacts. Review fixed +cached-discovery admission and the missing fresh-review route after another owned +request resolves an attempt. No unresolved critical/major finding remains for +this local restore-journal UI/discovery scope. The cognitive-complexity skill was +consulted; its dispatcher does not support Rust. Native Clippy required boxing the +large review reply and splitting a fixture's source-observation wait (16 versus +15); final checks pass without allowances. Test helper reborrows, a clone of a Copy +state and an initially missed guard edit were corrected before final verification. + +Verification on local macOS arm64: the full four-thread TUI library suite passed +669 tests with three configured ignores in 325.60 seconds. These include existing +rewind/undo/redo, composer, theme, accessibility and screen regressions. Subsequent +focused recovery checks passed five cases including the strengthened native +keep/rollback/unrelated-owner case (44.13 seconds), plus the final settled-attempt +fresh-review state test. The full HTTP suite passed all 79 tests, and the focused +runtime cancelled-request/discovery case passed. The actual CLI partial-kill +keep/rollback test passed in 86.26 seconds, including post-restart and foreign +request discovery. Final all-target locked server/TUI Clippy passed with Rust +warnings denied. All Cargo checks/builds/tests used CARGO_INCREMENTAL=0 and --locked. + +SDK typechecking passed. The initial SDK run passed 135 with ten real-binary cases +skipped because CYBER_BIN was absent; rerunning with the built binary passed 143 +with two platform skips. TUI ignores include the isolated pure-mode child fixture +and two Linux temporary-protected-file cases that avoid native vault writes on +macOS. Native C sandbox_init/sandbox_free_error deprecation and oversized unwind +linker warnings were observed without unchanged-main attribution. OpenAPI generation, +SDK consistency, format/diff, pinned strict OpenSpec (117 items) and spec lint +(0 errors/25 warnings) passed. No native Linux/Windows execution, full workspace, +OS TUI-kill matrix or source-bound CI is claimed. Preparation clients, actual managed +Git source/runtime integration, partial-constructor recovery, reference/reader/ +lease-aware reclamation and full cap/platform/client acceptance remain open. No +parent task, issue, Git availability guard or delivery status is promoted; no +PR/push is made by this local continuation. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 53afbfd09..626ce149f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -748,3 +748,16 @@ for the exact request. Text and JSON SHALL distinguish filesystem completion, durable runtime state and observed native acknowledgement, including unavailable historical evidence, conflicts and retained created parents. These clients SHALL NOT manufacture origin proof or bypass unavailable backend/runtime admission. + +### Recovery request discovery and terminal panel + +A bounded authenticated read-only request-discovery operation SHALL expose retained +confirmation identities for an exact Session creation and journal. It SHALL verify +native request ownership and SHALL NOT inspect source files, acknowledge work or +replay a request. A fresh terminal SHALL use discovery to find previously reserved +requests after reply loss; client memory SHALL NOT be the sole retained identity. +The rewind panel SHALL link to recovery with `C`, list journals and their retained +requests, review explicit keep/rollback selections, and require a second Enter for +the reviewed fingerprint. Request lookup and native acknowledgement reconciliation +SHALL remain separate explicit keys. Session/Location/UI-generation changes SHALL +invalidate stale replies without changing the composer or clearing native fences. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 87572b390..d3812d6b0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -358,3 +358,10 @@ - **THEN** confirmation SHALL require the exact fingerprint and an explicit stable request ID, and incomplete or malformed combinations SHALL fail before service startup - **AND** `sessions recovery-request` SHALL look up the exact retained request without replay; only explicit `--acknowledge` SHALL invoke metadata reconciliation - **AND** text/JSON SHALL distinguish current images, historical evidence availability, conflicts, durable state and observed native acknowledgement; stale/foreign responses or requests SHALL NOT authorize another Session/journal operation + +#### Scenario: A fresh terminal discovers the existing recovery request +- **GIVEN** a recovery confirmation reserved an exact durable request and its reply was lost +- **WHEN** the owner reopens rewind recovery in a fresh TUI +- **THEN** bounded read-only discovery SHALL identify requests for that exact Session creation and journal without native access, acknowledgement or replay +- **AND** the panel SHALL offer lookup of retained identities, keep native reconciliation explicit, preserve the composer/conversation and reject stale Session/Location/generation replies +- **AND** unavailable discovery or legacy proof SHALL NOT enable an unreviewed replacement attempt or clear native fences diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 085fe4bec..eda51ca91 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -472,3 +472,20 @@ stale/foreign request, repeat lookup and later owner-edit coverage now exercise these typed commands. This advances the restore-journal client portion of M7; preparation/TUI clients, actual managed Git integration, safe cleanup and full native/platform/client acceptance remain open. No parent task is closed. + +Terminal restore-recovery continuation: rewind now exposes C recovery with retained +journal/request selection, explicit keep/rollback review and second-Enter +confirmation, read-only request lookup and separate A acknowledgement metadata +reconciliation. Bounded server-side discovery uses the exact Session creation, +journal and native request-owner identity, so a fresh App can find a reserved +request after losing its reply. Failed discovery cannot authorize a fresh attempt; +legacy proof, pending/unknown outcomes and unrelated fences remain protected. +Actual production-App/native tests interrupt after a proven source replacement, +exercise both keep/rollback, preserve unrelated/later owner edits and the composer, +discard a successful reply, create a fresh TUI App and discover/lookup/reconcile +without inference or transcript movement. This is native cancellation plus a fresh +App, not an OS kill of the TUI process. Updated CLI service-kill acceptance also +checks discovery after server restart and foreign-Session filtering. Preparation +clients, ordinary managed Git source integration, partial construction, reclamation +and complete cut/platform acceptance remain open; M2–M8 and parent tasks are not +closed by this client continuation. diff --git a/sdk/openapi.json b/sdk/openapi.json index 43478425a..db1e22e08 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7633,6 +7633,20 @@ "directory" ] }, + "Data_Array_of_SnapshotRecoveryConfirmation": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SnapshotRecoveryConfirmation" + } + } + }, + "required": [ + "data" + ] + }, "Data_SnapshotRestoreReceipt": { "type": "object", "properties": { @@ -24840,6 +24854,55 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/recovery/requests": { + "get": { + "operationId": "v1.session.snapshotRecoveryRequests", + "summary": "Discover up to 4096 retained request identities for this Session creation and exact journal without native access, acknowledgement or replay", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "journal_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Data_Array_of_SnapshotRecoveryConfirmation" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}": { "get": { "operationId": "v1.session.snapshotRecoveryLookup", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index 4436698b6..1981ab56f 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1607,6 +1607,14 @@ export interface Operations { body: undefined; response: S.SnapshotRecoveryResult; }; + "v1.session.snapshotRecoveryRequests": { + path: { sessionID: string }; + query: { + journal_id?: QueryValue; + }; + body: undefined; + response: S.SnapshotRecoveryConfirmation[]; + }; "v1.session.snapshotRecoveryReview": { path: { sessionID: string }; query: Record; @@ -4806,6 +4814,18 @@ export const operations = { unwrap: true, stream: false, }, + "v1.session.snapshotRecoveryRequests": { + tag: "session", + name: "snapshotRecoveryRequests", + method: "GET", + path: "/api/v1/sessions/{sessionID}/rewind/recovery/requests", + pathParams: ["sessionID"], + query: ["journal_id"], + body: "none", + located: false, + unwrap: true, + stream: false, + }, "v1.session.snapshotRecoveryReview": { tag: "session", name: "snapshotRecoveryReview", @@ -6259,6 +6279,9 @@ export function createGroups(caller: Caller) { /** Look up a durable recovery request without native access, acknowledgement reconciliation, replay or inference (`GET /api/v1/sessions/{sessionID}/rewind/recovery/requests/{requestID}`) */ snapshotRecoveryLookup: (sessionID: string, requestID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecoveryLookup", { path: { sessionID, requestID } }, options), + /** Discover up to 4096 retained request identities for this Session creation and exact journal without native access, acknowledgement or replay (`GET /api/v1/sessions/{sessionID}/rewind/recovery/requests`) */ + snapshotRecoveryRequests: (sessionID: string, query?: Operations["v1.session.snapshotRecoveryRequests"]["query"], options?: RequestOptions): Promise => + caller.call("v1.session.snapshotRecoveryRequests", { path: { sessionID }, query }, options), /** Review retained snapshot effects without restoring, acknowledging native activity or starting inference (`POST /api/v1/sessions/{sessionID}/rewind/recovery/review`) */ snapshotRecoveryReview: (sessionID: string, body: Operations["v1.session.snapshotRecoveryReview"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecoveryReview", { path: { sessionID }, body }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index ea2696e50..274944114 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -910,6 +910,10 @@ export type Data_Array_of_SnapshotRecovery = { data: SnapshotRecovery[]; }; +export type Data_Array_of_SnapshotRecoveryConfirmation = { + data: SnapshotRecoveryConfirmation[]; +}; + export type Data_Array_of_Task = { data: Task[]; }; diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index 91f15efbf..8a0e2e03f 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -100,3 +100,15 @@ test("pre-effect cancellation acknowledgement retains origin proof without repla assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/preparations/op_fixture/requests/cancel%2Fwith%20space/acknowledge"); assert.ok(!calls[0]!.body); }); + +test("snapshot recovery request discovery is a scoped read without acknowledgement", async () => { + const request = { journal_id: "restore_one", action: "rollback" as const, + confirmation: "a".repeat(64), request_id: "retained/with space" }; + const { client, calls } = mockClient(() => json(200, { data: [request] })); + assert.deepEqual(await client.session.snapshotRecoveryRequests("ses_owner", { journal_id: request.journal_id }), [request]); + assert.equal(calls.length, 1); + assert.equal(calls[0]!.method, "GET"); + assert.equal(calls[0]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/recovery/requests"); + assert.equal(calls[0]!.url.searchParams.get("journal_id"), request.journal_id); + assert.ok(!calls[0]!.body); +}); From d8a7242de9623cdbb12a351cbf48790673b388a4 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 08:50:24 -0400 Subject: [PATCH 39/87] Add typed preparation recovery CLI commands --- crates/cyber-cli/src/commands/mod.rs | 1 + .../src/commands/preparation_recovery.rs | 360 ++++++++++++++++++ crates/cyber-cli/src/commands/recovery.rs | 6 +- crates/cyber-cli/src/commands/sessions.rs | 30 +- crates/cyber-cli/tests/branches_cli.rs | 123 ++++-- docs/cli.md | 38 ++ docs/rewind.md | 42 ++ .../implementation.md | 43 +++ .../managed-recovery.md | 13 + .../specs/snapshots-checkpoints/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 13 + 11 files changed, 648 insertions(+), 27 deletions(-) create mode 100644 crates/cyber-cli/src/commands/preparation_recovery.rs diff --git a/crates/cyber-cli/src/commands/mod.rs b/crates/cyber-cli/src/commands/mod.rs index b288ae43a..ed3edab47 100644 --- a/crates/cyber-cli/src/commands/mod.rs +++ b/crates/cyber-cli/src/commands/mod.rs @@ -29,6 +29,7 @@ pub mod permissions; pub mod plugin_config; pub mod plugins; pub mod pr; +mod preparation_recovery; pub mod providers; mod recovery; pub mod review; diff --git a/crates/cyber-cli/src/commands/preparation_recovery.rs b/crates/cyber-cli/src/commands/preparation_recovery.rs new file mode 100644 index 000000000..3bd7dde86 --- /dev/null +++ b/crates/cyber-cli/src/commands/preparation_recovery.rs @@ -0,0 +1,360 @@ +//! Cancellation uses exact exited-worker proof; missing intents never license clearing fences. +use super::{ + recovery::{data, fingerprint, request_id}, + sessions::{api, clean, encode, session_id}, +}; +use crate::{cli::GlobalArgs, error::CliError, output}; +use clap::Args; +use cyber_client::Client; +use cyber_server::runtime::{ + SnapshotPreparation, SnapshotPreparationConfirmation, SnapshotPreparationReceipt, + SnapshotPreparationReview, +}; +use serde_json::{Value, json}; + +#[derive(Debug, Args)] +pub struct PreparationArgs { + /// Session ID, exact title or unique ID prefix. + session: String, + /// Exact snapshot activity ID; omit to list retained preparation candidates. + #[arg(value_parser = activity_id)] + activity: Option, + /// Cancel only the exact reviewed preparation before admitted source effects. + #[arg(long, value_parser = fingerprint, requires_all = ["activity", "request_id"])] + confirm: Option, + /// Stable cancellation identity to retain for lookup after interruption or reply loss. + #[arg(long, value_parser = request_id, requires = "confirm")] + request_id: Option, +} +#[derive(Debug, Args)] +pub struct RequestArgs { + /// Session ID, exact title or unique ID prefix. + session: String, + /// Exact snapshot activity ID from preparation review. + #[arg(value_parser = activity_id)] + activity: String, + /// Exact retained cancellation request ID. + #[arg(value_parser = request_id)] + request_id: String, + /// Reconcile the committed receipt's native metadata without source-file access or replay. + #[arg(long)] + acknowledge: bool, +} +fn activity_id(value: &str) -> Result { + if value.starts_with("op_") + && value.len() <= 256 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + Ok(value.into()) + } else { + Err("expected an exact snapshot activity ID from preparation recovery".into()) + } +} +fn hint(session: &str, activity: &str, request: &str) -> String { + let ids = [session, activity, request]; + if ids.iter().flat_map(|id| id.chars()).any(char::is_control) { + return format!( + "Look up the retained cancellation with sessions recovery-preparation-request using the original IDs (JSON): {}; do not choose a new request ID to retry", + json!(ids) + ); + } + format!( + "Inspect the retained cancellation with cyber sessions recovery-preparation-request -- {} {} {}; do not choose a new request ID to retry", + shell_words::quote(session), + shell_words::quote(activity), + shell_words::quote(request) + ) +} +fn origin_matches_lifetime( + origin: Option<&cyber_server::runtime::SnapshotOrigin>, + lifetime: &cyber_server::runtime::SnapshotLifetimeBinding, +) -> bool { + origin.is_none_or(|origin| { + origin.version == 1 + && lifetime.version == 2 + && origin.session_id == lifetime.owner.session_id + && origin.activity_id == lifetime.owner.activity_id + && origin.session_created_id == lifetime.owner.session_created_id + && origin.worktree_id == lifetime.owner.worktree_id + && origin.binding.directory == lifetime.owner.directory + && origin.binding.root_identity == lifetime.root_identity + }) +} +fn review_response( + reply: Value, + session: &str, + activity: &str, +) -> Result { + let review: SnapshotPreparationReview = data(reply)?; + if review.lifetime.version != 2 + || review.lifetime.source_phase.is_none() + || review.activity_id != activity + || review.lifetime.owner.activity_id != activity + || review.lifetime.owner.session_id != session + || !origin_matches_lifetime(review.origin.as_ref(), &review.lifetime) + { + return Err(CliError::runtime( + "Preparation review lacks matching current pre-effect ownership evidence", + )); + } + Ok(review) +} +fn receipt_response( + reply: Value, + session: &str, + activity: &str, + request: &str, +) -> Result { + let receipt: SnapshotPreparationReceipt = data(reply)?; + if receipt.activity_id != activity + || receipt.request_id != request + || receipt.lifetime.owner.activity_id != activity + || receipt.lifetime.owner.session_id != session + || receipt.source_admitted == Some(true) + || (receipt.origin.is_some() && receipt.source_admitted != Some(false)) + || !origin_matches_lifetime(receipt.origin.as_ref(), &receipt.lifetime) + { + return Err(CliError::runtime( + "Cancellation receipt does not match the selected pre-effect owner and request", + )); + } + Ok(receipt) +} +pub async fn run( + client: &Client, + args: PreparationArgs, + global: &GlobalArgs, +) -> Result<(), CliError> { + let session = session_id(client, &args.session).await?; + let base = format!("/sessions/{}/rewind/preparations", encode(&session)); + let Some(activity) = args.activity else { + let list: Vec = data(client.get(&base).await.map_err(api)?)?; + return list_output(&list, global); + }; + let base = format!("{base}/{}", encode(&activity)); + if let Some(confirmation) = args.confirm { + let request_id = args + .request_id + .ok_or_else(|| CliError::usage("Preparation confirmation requires --request-id"))?; + let lookup = hint(&session, &activity, &request_id); + let body = SnapshotPreparationConfirmation { + confirmation, + request_id, + }; + let reply = client + .post(&format!("{base}/confirm"), json!(&body)) + .await + .map_err(|error| api(error).with_hint(&lookup))?; + let receipt = receipt_response(reply, &session, &activity, &body.request_id) + .map_err(|error| error.with_hint(&lookup))?; + if receipt.confirmation != body.confirmation { + return Err(CliError::runtime( + "Cancellation reply differs from the confirmed fingerprint", + ) + .with_hint(lookup)); + } + return receipt_output(&receipt, global, false); + } + let review = review_response( + client + .post(&format!("{base}/review"), json!({})) + .await + .map_err(api)?, + &session, + &activity, + )?; + review_output(&review, global) +} +pub async fn request( + client: &Client, + args: RequestArgs, + global: &GlobalArgs, +) -> Result<(), CliError> { + let session = session_id(client, &args.session).await?; + let path = format!( + "/sessions/{}/rewind/preparations/{}/requests/{}", + encode(&session), + encode(&args.activity), + encode(&args.request_id) + ); + let lookup = hint(&session, &args.activity, &args.request_id); + let reply = if args.acknowledge { + client.post(&format!("{path}/acknowledge"), json!({})).await + } else { + client.get(&path).await + } + .map_err(|error| api(error).with_hint(&lookup))?; + let receipt = receipt_response(reply, &session, &args.activity, &args.request_id) + .map_err(|error| error.with_hint(&lookup))?; + receipt_output(&receipt, global, args.acknowledge) +} +fn list_output(list: &[SnapshotPreparation], global: &GlobalArgs) -> Result<(), CliError> { + if output::is_json(global.format) { + return output::json(&list); + } + if list.is_empty() { + println!("No retained pre-effect preparation candidates."); + } + for item in list { + println!( + "{} · status={} · lifetime evidence={}", + clean(&item.activity_id), + clean(&item.status), + item.lifetime_available + ); + } + println!( + "Listing is not cancellation authority. Review an exact activity; live workers and unavailable historical proof refuse." + ); + Ok(()) +} +fn review_output(review: &SnapshotPreparationReview, global: &GlobalArgs) -> Result<(), CliError> { + if output::is_json(global.format) { + return output::json(review); + } + println!( + "Preparation cancellation review: {}", + clean(&review.activity_id) + ); + println!( + "Session: {}; Location: {:?}; lifetime version: {}", + clean(&review.lifetime.owner.session_id), + review.lifetime.owner.directory, + review.lifetime.version + ); + println!( + "Native sequence: {}; Session sequence: {}", + review.native_sequence, review.session_sequence + ); + println!( + "Published origin: {}; acquired checkout digest: {}", + review.origin.is_some(), + review + .checkout_resources_digest + .as_deref() + .map(clean) + .unwrap_or_else(|| "unavailable historical proof".into()) + ); + println!("{}", clean(&review.warning)); + println!( + "No source files or transcript changed. To cancel this reviewed preparation, repeat with --confirm {} --request-id ", + clean(&review.confirmation) + ); + Ok(()) +} +fn receipt_output( + receipt: &SnapshotPreparationReceipt, + global: &GlobalArgs, + reconciled: bool, +) -> Result<(), CliError> { + if output::is_json(global.format) { + return output::json(receipt); + } + println!( + "Committed preparation cancellation: {} · request {:?}", + clean(&receipt.activity_id), + receipt.request_id + ); + println!( + "Session: {}; Location: {:?}", + clean(&receipt.lifetime.owner.session_id), + receipt.lifetime.owner.directory + ); + println!( + "Lifetime version: {}; acquired checkout digest: {}", + receipt.lifetime.version, + receipt + .checkout_resources_digest + .as_deref() + .map(clean) + .unwrap_or_else(|| "unavailable historical proof".into()) + ); + if reconciled { + println!("Acknowledgement reconciliation completed for this receipt."); + } else { + println!( + "This receipt establishes durable cancellation; this response alone does not observe native acknowledgement. Use recovery-preparation-request --acknowledge to reconcile explicitly." + ); + } + println!( + "Files and transcript are preserved. Independent process/checkout uncertainty and recovery evidence remain protected." + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[test] + fn unreviewed_or_malformed_cancellations_fail_before_startup() { + let hash = "a".repeat(64); + for tail in [ + vec!["--confirm", &hash, "--request-id", "one"], + vec!["op_one", "--confirm", &hash], + vec!["op_one", "--request-id", "one"], + vec!["op_one", "--confirm", "yes", "--request-id", "one"], + vec!["../op_one"], + vec!["op_one", "--force"], + vec!["op_one", "--confirm", &hash, "--request-id", ".."], + ] { + let mut args = vec!["cyber", "sessions", "recovery-preparation", "ses_one"]; + args.extend(tail); + assert!(crate::cli::Cli::try_parse_from(&args).is_err(), "{args:?}"); + } + for tail in [ + vec![], + vec!["op_one"], + vec!["op_one", "--confirm", &hash, "--request-id", "one / #?%"], + ] { + let mut args = vec!["cyber", "sessions", "recovery-preparation", "ses_one"]; + args.extend(tail); + crate::cli::Cli::try_parse_from(args).unwrap(); + } + crate::cli::Cli::try_parse_from([ + "cyber", + "sessions", + "recovery-preparation-request", + "--acknowledge", + "--", + "ses_one", + "op_one", + "-request", + ]) + .unwrap(); + assert!(!hint("ses_one", "op_one", "line\n\x1b[31m").contains(['\n', '\x1b'])); + } + fn lifetime() -> Value { + json!({"version":2,"owner":{"activity_id":"op_one","session_id":"ses_one","session_created_id":"created","directory":"/project","worktree_id":null},"root_identity":"root","namespace_identity":"namespace","directory_identity":"directory","file_identity":"file","source_phase":"phase"}) + } + #[test] + fn receipts_and_reviews_must_match_the_selected_pre_effect_owner() { + let review = json!({"data":{"activity_id":"op_one","lifetime":lifetime(),"native_sequence":0,"session_sequence":1,"confirmation":"a".repeat(64),"warning":"independent uncertainty remains","origin":null,"checkout_resources_digest":"d".repeat(64)}}); + review_response(review.clone(), "ses_one", "op_one").unwrap(); + let mut unsupported = review.clone(); + unsupported["data"]["lifetime"]["version"] = 1.into(); + assert!(review_response(unsupported, "ses_one", "op_one").is_err()); + let mut missing_phase = review.clone(); + missing_phase["data"]["lifetime"]["source_phase"] = Value::Null; + assert!(review_response(missing_phase, "ses_one", "op_one").is_err()); + assert!(review_response(review.clone(), "ses_foreign", "op_one").is_err()); + assert!(review_response(review, "ses_one", "op_foreign").is_err()); + let mut receipt = json!({"data":{"activity_id":"op_one","request_id":"request / #?%","confirmation":"a".repeat(64),"lifetime":lifetime(),"native_sequence":0,"session_sequence":1,"origin":null,"source_admitted":false,"checkout_resources_digest":"d".repeat(64)}}); + receipt_response(receipt.clone(), "ses_one", "op_one", "request / #?%").unwrap(); + assert!( + receipt_response(receipt.clone(), "ses_foreign", "op_one", "request / #?%").is_err() + ); + assert!( + receipt_response(receipt.clone(), "ses_one", "op_foreign", "request / #?%").is_err() + ); + assert!(receipt_response(receipt.clone(), "ses_one", "op_one", "foreign").is_err()); + receipt["data"]["origin"] = json!({"version":1,"session_id":"ses_foreign","session_created_id":"created","activity_id":"op_one","worktree_id":null,"binding":{"directory":"/project","root_identity":"root","namespace_identity":"journal_namespace","journal_id":"restore_one","journal_identity":"journal"}}); + assert!(receipt_response(receipt.clone(), "ses_one", "op_one", "request / #?%").is_err()); + receipt["data"]["origin"] = Value::Null; + receipt["data"]["source_admitted"] = true.into(); + assert!(receipt_response(receipt, "ses_one", "op_one", "request / #?%").is_err()); + } +} diff --git a/crates/cyber-cli/src/commands/recovery.rs b/crates/cyber-cli/src/commands/recovery.rs index 4ade06581..9c8505a27 100644 --- a/crates/cyber-cli/src/commands/recovery.rs +++ b/crates/cyber-cli/src/commands/recovery.rs @@ -66,7 +66,7 @@ fn journal_id(value: &str) -> Result { Err("expected an exact restore journal ID from sessions recovery".into()) } } -fn fingerprint(value: &str) -> Result { +pub(super) fn fingerprint(value: &str) -> Result { if value.len() == 64 && value .bytes() @@ -77,7 +77,7 @@ fn fingerprint(value: &str) -> Result { Err("expected the 64-character lowercase fingerprint from recovery review".into()) } } -fn request_id(value: &str) -> Result { +pub(super) fn request_id(value: &str) -> Result { if matches!(value, "." | "..") { return Err("a dot path segment cannot be addressed as a recovery request; choose a stable named ID".into()); } @@ -87,7 +87,7 @@ fn request_id(value: &str) -> Result { Err("recovery request ID must contain 1-256 bytes without NUL".into()) } } -fn data(reply: Value) -> Result { +pub(super) fn data(reply: Value) -> Result { serde_json::from_value(reply["data"].clone()) .map_err(|error| CliError::runtime(format!("Invalid recovery response: {error}"))) } diff --git a/crates/cyber-cli/src/commands/sessions.rs b/crates/cyber-cli/src/commands/sessions.rs index d7484009b..19df637d9 100644 --- a/crates/cyber-cli/src/commands/sessions.rs +++ b/crates/cyber-cli/src/commands/sessions.rs @@ -80,6 +80,10 @@ pub enum SessionsCmd { Recovery(super::recovery::RecoveryArgs), /// Look up a retained recovery request; optionally reconcile its native acknowledgement. RecoveryRequest(super::recovery::RequestArgs), + /// Review or cancel an exited snapshot preparation before source effects were admitted. + RecoveryPreparation(super::preparation_recovery::PreparationArgs), + /// Look up a committed preparation cancellation; optionally reconcile its native metadata. + RecoveryPreparationRequest(super::preparation_recovery::RequestArgs), /// Set or remove a branch label. Label { /// ID, exact title or unique ID prefix. @@ -203,13 +207,35 @@ async fn dispatch(client: &Client, cmd: SessionsCmd, global: &GlobalArgs) -> Res target, confirm, } => super::rewind::run(client, &session, &to, target, confirm, global).await, - SessionsCmd::Recovery(args) => super::recovery::run(client, args, global).await, - SessionsCmd::RecoveryRequest(args) => super::recovery::request(client, args, global).await, + recovery @ (SessionsCmd::Recovery(_) + | SessionsCmd::RecoveryRequest(_) + | SessionsCmd::RecoveryPreparation(_) + | SessionsCmd::RecoveryPreparationRequest(_)) => { + dispatch_recovery(client, recovery, global).await + } SessionsCmd::Other(_) => unreachable!("handled before connecting"), branch => dispatch_branch(client, branch, global).await, } } +async fn dispatch_recovery( + client: &Client, + command: SessionsCmd, + global: &GlobalArgs, +) -> Result<(), CliError> { + match command { + SessionsCmd::Recovery(args) => super::recovery::run(client, args, global).await, + SessionsCmd::RecoveryRequest(args) => super::recovery::request(client, args, global).await, + SessionsCmd::RecoveryPreparation(args) => { + super::preparation_recovery::run(client, args, global).await + } + SessionsCmd::RecoveryPreparationRequest(args) => { + super::preparation_recovery::request(client, args, global).await + } + _ => unreachable!("only recovery commands are dispatched here"), + } +} + /// The branch commands: `tree`, `checkout` and `label`. async fn dispatch_branch( client: &Client, diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index bb0dff702..3e76ccd50 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -818,39 +818,46 @@ fn cancelled_pre_effect_preparation(published: bool) { std::fs::write(&target, "later owner bytes").unwrap(); std::fs::remove_file(repo.join("bulk_0000")).unwrap(); std::os::unix::fs::symlink(&target, repo.join("bulk_0000")).unwrap(); - let list = - json(&env.cyber(&["api", "GET", &format!("/sessions/{id}/rewind/preparations")]))["data"] - .clone(); + let list = json(&env.cyber(&["sessions", "recovery-preparation", &id, "--format", "json"])); assert_eq!(list[0]["activity_id"], activity); assert_eq!(list[0]["lifetime_available"], true); - let route = format!("/sessions/{id}/rewind/preparations/{activity}"); - let review = - json(&env.cyber(&["api", "POST", &format!("{route}/review"), "-d", "{}"]))["data"].clone(); - let body = - json!({"confirmation":review["confirmation"],"request_id":"before-origin"}).to_string(); - let receipt = - json(&env.cyber(&["api", "POST", &format!("{route}/confirm"), "-d", &body]))["data"] - .clone(); + let review = json(&env.cyber(&[ + "sessions", + "recovery-preparation", + &id, + &activity, + "--format", + "json", + ])); + let request_id = "before-origin/ owner #?%"; + let body = json!({"confirmation":review["confirmation"],"request_id":request_id}); + assert_preparation_client_refusals(&env, &id, &activity, &body); + let receipt = json(&preparation_confirm_cli(&env, &id, &activity, &body)); assert_preparation_resource_digest(&review, &receipt, published); - let acknowledge = format!("{route}/requests/before-origin/acknowledge"); assert_eq!( - json(&env.cyber(&["api", "POST", &acknowledge]))["data"], + json(&preparation_request_cli( + &env, &id, &activity, request_id, true + )), receipt ); if published { assert_preparation_tombstone(&env, &receipt); } kill_registered_service(&env); - let repeated = - json(&env.cyber(&["api", "POST", &format!("{route}/confirm"), "-d", &body]))["data"] - .clone(); - assert_eq!(repeated, receipt); - let lookup = - json(&env.cyber(&["api", "GET", &format!("{route}/requests/before-origin")]))["data"] - .clone(); - assert_eq!(lookup, receipt); assert_eq!( - json(&env.cyber(&["api", "POST", &acknowledge]))["data"], + json(&preparation_confirm_cli(&env, &id, &activity, &body)), + receipt + ); + assert_eq!( + json(&preparation_request_cli( + &env, &id, &activity, request_id, false + )), + receipt + ); + assert_eq!( + json(&preparation_request_cli( + &env, &id, &activity, request_id, true + )), receipt ); assert!( @@ -869,6 +876,78 @@ fn cancelled_pre_effect_preparation(published: bool) { env.exec(Some(&id), "after preparation cancellation"); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn preparation_confirm_cli(env: &Env, session: &str, activity: &str, body: &Value) -> Output { + env.cyber(&[ + "sessions", + "recovery-preparation", + session, + activity, + "--confirm", + body["confirmation"].as_str().unwrap(), + "--request-id", + body["request_id"].as_str().unwrap(), + "--format", + "json", + ]) +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn preparation_request_cli( + env: &Env, + session: &str, + activity: &str, + request: &str, + acknowledge: bool, +) -> Output { + let mut args = vec![ + "sessions", + "recovery-preparation-request", + session, + activity, + request, + "--format", + "json", + ]; + if acknowledge { + args.push("--acknowledge"); + } + env.cyber(&args) +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_preparation_client_refusals(env: &Env, session: &str, activity: &str, body: &Value) { + let mut stale = body.clone(); + stale["confirmation"] = "0".repeat(64).into(); + let refused = preparation_confirm_cli(env, session, activity, &stale); + assert_eq!(refused.status.code(), Some(1)); + assert!(String::from_utf8_lossy(&refused.stderr).contains("recovery-preparation-request")); + let foreign = json(&env.cyber(&[ + "api", + "POST", + "/sessions", + "-d", + "{\"title\":\"foreign preparation\"}", + ])); + let foreign_id = foreign["data"]["id"].as_str().unwrap(); + assert!( + !env.cyber(&[ + "sessions", + "recovery-preparation", + foreign_id, + activity, + "--format", + "json" + ]) + .status + .success() + ); + assert!( + !preparation_confirm_cli(env, foreign_id, activity, body) + .status + .success() + ); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_preparation_resource_digest(review: &Value, receipt: &Value, published: bool) { assert_eq!( diff --git a/docs/cli.md b/docs/cli.md index 33235e1f8..d5d547d3b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -320,6 +320,8 @@ cyber sessions [OPTIONS] - [`rewind`](#cyber-sessions-rewind): Preview a rewind before a user message; apply only with its reviewed fingerprint - [`recovery`](#cyber-sessions-recovery): List retained restore journals, review keep/rollback, or confirm a reviewed recovery - [`recovery-request`](#cyber-sessions-recovery-request): Look up a retained recovery request; optionally reconcile its native acknowledgement +- [`recovery-preparation`](#cyber-sessions-recovery-preparation): Review or cancel an exited snapshot preparation before source effects were admitted +- [`recovery-preparation-request`](#cyber-sessions-recovery-preparation-request): Look up a committed preparation cancellation; optionally reconcile its native metadata - [`label`](#cyber-sessions-label): Set or remove a branch label - [`export`](#cyber-sessions-export): Write a Session as JSON (default), Markdown (--format md) or one self-contained HTML page (--format html): its info, messages, tool calls, goals and workflow runs - [`import`](#cyber-sessions-import): Import a JSON export (`cyber sessions export`) as a new Session in this Location, with fresh IDs and its attachments, which are verified before anything is written @@ -461,6 +463,42 @@ cyber sessions recovery-request [OPTIONS] - `--acknowledge`: Reconcile native acknowledgement of the committed request; never replay source work. +## `cyber sessions recovery-preparation` + +Review or cancel an exited snapshot preparation before source effects were admitted + +```text +cyber sessions recovery-preparation [OPTIONS] [ACTIVITY] +``` + +**Arguments:** + +- ``: Session ID, exact title or unique ID prefix. +- `[ACTIVITY]`: Exact snapshot activity ID; omit to list retained preparation candidates. + +**Options:** + +- `--confirm `: Cancel only the exact reviewed preparation before admitted source effects. +- `--request-id `: Stable cancellation identity to retain for lookup after interruption or reply loss. + +## `cyber sessions recovery-preparation-request` + +Look up a committed preparation cancellation; optionally reconcile its native metadata + +```text +cyber sessions recovery-preparation-request [OPTIONS] +``` + +**Arguments:** + +- ``: Session ID, exact title or unique ID prefix. +- ``: Exact snapshot activity ID from preparation review. +- ``: Exact retained cancellation request ID. + +**Options:** + +- `--acknowledge`: Reconcile the committed receipt's native metadata without source-file access or replay. + ## `cyber sessions label` Set or remove a branch label diff --git a/docs/rewind.md b/docs/rewind.md index 81fbbc2b5..b14e9ad56 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -442,3 +442,45 @@ filters to the current Session creation, verifies request-owner identities and returns at most 4096 retained confirmation records. Overflow refuses explicitly without deleting history. Discovery does not inspect source files, acknowledge native work or start inference. The SDK exposes `session.snapshotRecoveryRequests`. + +## Recovering an exited pre-effect preparation + +A worker can stop before it publishes an intent or admits source effects. Its +missing journal alone does not prove that cancellation is safe. List retained +candidates and review an exact activity: + +```sh +cyber sessions recovery-preparation ses_example +cyber sessions recovery-preparation ses_example op_example +``` + +Listing shows durable status and whether lifetime metadata exists; it does not +establish that the worker exited. Review acquires the exact retained native +lifetime/checkout evidence and refuses a live worker, missing historical proof, +stale ownership or possible admitted effects. It shows the published-origin flag, +versions/sequences, complete checkout-set digest when available and the protocol +warning. Nothing is cancelled by review. + +Confirm only a reviewed fingerprint and retain your stable cancellation ID: + +```sh +cyber sessions recovery-preparation ses_example op_example \\ + --confirm --request-id cancel-one +cyber sessions recovery-preparation-request ses_example op_example cancel-one +cyber sessions recovery-preparation-request ses_example op_example cancel-one --acknowledge +``` + +The second command looks up the exact committed cancellation without replay; the +third explicitly reconciles that receipt's native metadata. Repeating the original +confirmation with the same request ID/content also returns its retained receipt. +A receipt lookup does not establish observation of native acknowledgement. Errors +after confirmation include the exact lookup command, with quoted IDs. Never choose +a new request ID to retry an uncertain outcome. All forms support `--format json`; +malformed/incomplete confirmation options fail before service startup. + +Cancellation preserves files, transcript, retained intents, private payloads and +independent process/checkout uncertainty. Older receipts can remain inspectable +without gaining newer native acknowledgement authority. If source effects may +have been admitted, use the journal's keep/rollback review instead. No force-clear +or evidence deletion option exists. The TUI preparation path and remaining partial +constructor/native-platform acceptance are still unfinished. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 7b56821e4..dcdcd2830 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -2981,3 +2981,46 @@ Git source/runtime integration, partial-constructor recovery, reference/reader/ lease-aware reclamation and full cap/platform/client acceptance remain open. No parent task, issue, Git availability guard or delivery status is promoted; no PR/push is made by this local continuation. + +## Typed preparation recovery CLI (2026-11-07) + +`sessions recovery-preparation` now lists candidates or reviews an exact activity. +Cancellation requires both its fingerprint and caller-retained request ID. The +separate `sessions recovery-preparation-request` performs committed receipt lookup; +explicit `--acknowledge` invokes existing native metadata reconciliation. No server +operation, native admission rule or journal encoding changes. Shared fingerprint, +request-ID and typed-response helpers retain their existing validation. Replies +must match Session/activity/lifetime and any published origin's creation, worktree, +directory and root identity. Namespace identities with different roles are not +conflated. Unsupported review lifetime/phase evidence and admitted-effect receipts +refuse. Historical receipts retain read semantics without acquiring new proof. + +Text distinguishes durable cancellation from native acknowledgement observation, +quotes paths/IDs, reports unavailable checkout proof and preserves independent +uncertainty. Ambiguous confirmation errors retain an exact shell-quoted or +JSON-escaped lookup hint. No force option exists. The operator guide and generated +CLI reference are synchronized. Grouped recovery dispatch keeps the Session +router readable; the existing deletion handler remains unchanged. + +Both actual pre-origin and published-origin-before-source service-kill cases now +use typed listing/review/confirmation, repeated confirmation, request lookup after +restart and explicit reconciliation. They verify stale/foreign refusal, exact +resource digests/tombstones, later owner symlinks/files, unchanged messages and no +inference. Final local macOS arm64 checks passed: all 86 CLI unit tests, all 17 +serial branch/recovery cases (187.37 seconds), all-target locked CLI Clippy, +format/diff, generated CLI/SDK consistency and pinned strict OpenSpec (117 items). +The two focused native cuts also passed separately in 61.63 seconds. Spec lint +reported 0 errors/25 warnings, without unchanged-main attribution. Every Cargo +build/check/test/Clippy used CARGO_INCREMENTAL=0 and --locked. + +The code-review skill covered parser-before-startup refusal, exact response scope, +legacy evidence, replay/lookup separation, native acknowledgement limits and all +changed tests/docs. No unresolved critical/major finding remains in this CLI +scope. The cognitive-complexity skill was applied using native Rust Clippy because +its dispatcher does not support Rust; grouping recovery routing resolved 17/15 +complexity without allowances. C deprecation and oversized unwind linker warnings +were observed without origin attribution. No separate SDK runtime, TUI/server/full +workspace, Linux/Windows native or source-bound CI checks ran for this command-only +continuation. Preparation TUI, partial constructors, managed Git source integration, +reference-aware reclamation and broader acceptance remain open; no parent task, +issue or Git availability guard is promoted. No PR/push is made by this local work. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 626ce149f..53a3671f0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -761,3 +761,16 @@ requests, review explicit keep/rollback selections, and require a second Enter f the reviewed fingerprint. Request lookup and native acknowledgement reconciliation SHALL remain separate explicit keys. Session/Location/UI-generation changes SHALL invalidate stale replies without changing the composer or clearing native fences. + +### Typed preparation recovery commands + +`sessions recovery-preparation [activity]` SHALL list candidates when no +activity is selected and otherwise obtain a read-only eligibility review. Explicit +`--confirm --request-id ` SHALL consume only the exact +existing positive pre-effect cancellation protocol. The separate +`sessions recovery-preparation-request ` SHALL +perform committed receipt lookup; explicit `--acknowledge` SHALL reconcile native +metadata through the existing route. Replies SHALL match the selected Session, +activity and lifetime, and lookup SHALL NOT imply observed native acknowledgement. +No missing journal, historical lifetime or completed filesystem work SHALL be +promoted into cancellation authority by these clients. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index d3812d6b0..ad793a29b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -365,3 +365,9 @@ - **THEN** bounded read-only discovery SHALL identify requests for that exact Session creation and journal without native access, acknowledgement or replay - **AND** the panel SHALL offer lookup of retained identities, keep native reconciliation explicit, preserve the composer/conversation and reject stale Session/Location/generation replies - **AND** unavailable discovery or legacy proof SHALL NOT enable an unreviewed replacement attempt or clear native fences + +#### Scenario: Typed preparation cancellation preserves exact pre-effect ownership +- **WHEN** the owner lists or reviews an exited preparation through `sessions recovery-preparation` +- **THEN** cancellation SHALL require its exact reviewed fingerprint and stable request ID, with malformed/incomplete options refused before service startup +- **AND** replies SHALL match the selected Session/activity/lifetime and any published origin; source-admitted, mismatched or unsupported review evidence SHALL NOT be presented as cancellation authority +- **AND** `sessions recovery-preparation-request` SHALL look up the committed receipt without replay, keep acknowledgement reconciliation explicit, and preserve files, transcript, evidence and independent process/checkout uncertainty diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index eda51ca91..89cb45709 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -489,3 +489,16 @@ checks discovery after server restart and foreign-Session filtering. Preparation clients, ordinary managed Git source integration, partial construction, reclamation and complete cut/platform acceptance remain open; M2–M8 and parent tasks are not closed by this client continuation. + +Typed preparation CLI continuation: `sessions recovery-preparation` lists and +reviews exact candidates, then requires a fingerprint and stable request ID for +pre-effect cancellation. `sessions recovery-preparation-request` provides receipt +lookup and explicit native metadata reconciliation. Replies bind selected +Session/activity/lifetime and published-origin context; unsupported review proof +or admitted effects refuse, while historical receipts retain their separate read +semantics. Real CLI pre-origin and published-origin-before-source kill cases now +use these typed commands for cancellation, restart lookup and reconciliation, +including stale/foreign refusal and preservation of later owner symlinks/files. +This advances the preparation CLI portion of M7; preparation TUI, partial +construction, managed Git integration, safe reclamation and complete acceptance +remain open. No parent task is closed. From 55d746e7866acf7391544af03ac0b16311f366e1 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 09:13:53 -0400 Subject: [PATCH 40/87] Add terminal preparation recovery and receipt discovery --- crates/cyber-cli/tests/branches_cli.rs | 58 ++++ crates/cyber-server/src/http/openapi.rs | 4 +- crates/cyber-server/src/http/sessions.rs | 35 ++- .../src/runtime/snapshot_preparation.rs | 66 ++++- .../src/runtime/snapshot_work_tests.rs | 43 +++ crates/cyber-server/tests/http.rs | 14 + crates/cyber-tui/src/preparation_recovery.rs | 239 ++++++++++++++++ .../src/preparation_recovery_client.rs | 62 +++++ .../src/preparation_recovery_tests.rs | 255 ++++++++++++++++++ .../src/preparation_recovery_view.rs | 99 +++++++ crates/cyber-tui/src/recovery.rs | 18 +- crates/cyber-tui/src/recovery_client.rs | 9 +- crates/cyber-tui/src/recovery_view.rs | 10 +- docs/rewind.md | 25 +- .../implementation.md | 54 ++++ .../managed-recovery.md | 13 + .../specs/tui/spec.md | 9 + .../complete-daily-coding-surfaces/tasks.md | 15 ++ sdk/openapi.json | 170 ++++++++---- sdk/typescript/src/generated/operations.ts | 31 ++- sdk/typescript/src/generated/types.ts | 5 + sdk/typescript/test/rewind.test.ts | 10 + 22 files changed, 1179 insertions(+), 65 deletions(-) create mode 100644 crates/cyber-tui/src/preparation_recovery.rs create mode 100644 crates/cyber-tui/src/preparation_recovery_client.rs create mode 100644 crates/cyber-tui/src/preparation_recovery_tests.rs create mode 100644 crates/cyber-tui/src/preparation_recovery_view.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 3e76ccd50..de25286a1 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -844,6 +844,7 @@ fn cancelled_pre_effect_preparation(published: bool) { assert_preparation_tombstone(&env, &receipt); } kill_registered_service(&env); + assert_preparation_history(&env, &id, &activity, &receipt); assert_eq!( json(&preparation_confirm_cli(&env, &id, &activity, &body)), receipt @@ -876,6 +877,41 @@ fn cancelled_pre_effect_preparation(published: bool) { env.exec(Some(&id), "after preparation cancellation"); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_preparation_history(env: &Env, session: &str, activity: &str, receipt: &Value) { + let current = json(&env.cyber(&[ + "api", + "GET", + &format!("/sessions/{session}/rewind/preparations"), + ])); + assert!( + current["data"] + .as_array() + .unwrap() + .iter() + .all(|candidate| candidate["activity_id"] != activity) + ); + let history = json(&env.cyber(&[ + "api", + "GET", + &format!("/sessions/{session}/rewind/preparations?include_cancelled=true"), + ])); + assert!( + history["data"] + .as_array() + .unwrap() + .iter() + .any(|candidate| candidate["activity_id"] == activity + && candidate["status"] == "settled") + ); + let discovered = json(&env.cyber(&[ + "api", + "GET", + &format!("/sessions/{session}/rewind/preparations/{activity}/receipt"), + ])); + assert_eq!(&discovered["data"], receipt); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn preparation_confirm_cli(env: &Env, session: &str, activity: &str, body: &Value) -> Output { env.cyber(&[ @@ -929,6 +965,28 @@ fn assert_preparation_client_refusals(env: &Env, session: &str, activity: &str, "{\"title\":\"foreign preparation\"}", ])); let foreign_id = foreign["data"]["id"].as_str().unwrap(); + assert!( + !env.cyber(&[ + "api", + "GET", + &format!("/sessions/{foreign_id}/rewind/preparations/{activity}/receipt") + ]) + .status + .success() + ); + let foreign_history = json(&env.cyber(&[ + "api", + "GET", + &format!("/sessions/{foreign_id}/rewind/preparations?include_cancelled=true"), + ])); + assert!( + foreign_history["data"] + .as_array() + .unwrap() + .iter() + .all(|candidate| candidate["activity_id"] != activity) + ); + assert!( !env.cyber(&[ "sessions", diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index ac5f0904c..0a24868ee 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -784,7 +784,9 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { op("get", "/sessions/{sessionID}/rewind/restores/{activityID}", "v1.session.snapshotRestoreLookup", "Read the exact committed ordinary restore receipt without native access or replay") .ok::>(g), op("get", "/sessions/{sessionID}/rewind/preparations", "v1.session.snapshotPreparations", "List retained preparation candidates without native access; review checks eligibility") - .ok::>>(g), + .query(&["include_cancelled"]).ok::>>(g), + op("get", "/sessions/{sessionID}/rewind/preparations/{activityID}/receipt", "v1.session.snapshotPreparationReceipt", "Discover the exact committed preparation cancellation receipt without source access or native acknowledgement; absence is not pre-effect proof") + .ok::(g), op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/review", "v1.session.snapshotPreparationReview", "Review cancellation of an exited preparation using its exact native lifetime proof") .ok::>(g), op("post", "/sessions/{sessionID}/rewind/preparations/{activityID}/confirm", "v1.session.snapshotPreparationConfirm", "Cancel only a fingerprint-confirmed pre-effect snapshot preparation without moving files or transcript") diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index 0d1db9b1e..d731fb4a1 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -378,6 +378,10 @@ pub fn routes() -> Router { &format!("{s}/rewind/preparations"), get(snapshot_preparations), ) + .route( + &format!("{s}/rewind/preparations/{{activity_id}}/receipt"), + get(snapshot_preparation_receipt), + ) .route( &format!("{s}/rewind/preparations/{{activity_id}}/review"), post(snapshot_preparation_review), @@ -1341,12 +1345,41 @@ async fn snapshot_restore_acknowledge( .await?, })) } +#[derive(Default, Deserialize)] +struct PreparationsQuery { + #[serde(default)] + include_cancelled: bool, +} async fn snapshot_preparations( State(state): State, Path(id): Path, + Query(query): Query, ) -> Result>>> { Ok(Json(Data { - data: state.runtime.snapshot_preparations(&id).await?, + data: state + .runtime + .snapshot_preparations_with_history(&id, query.include_cancelled) + .await?, + })) +} +/// A discovery response always contains `data`, including an explicit absent receipt. +#[derive(Serialize, JsonSchema)] +#[schemars(transform = preparation_receipt_required)] +pub(super) struct PreparationReceiptData { + data: Option, +} +fn preparation_receipt_required(schema: &mut schemars::Schema) { + schema.insert("required".into(), serde_json::json!(["data"])); +} +async fn snapshot_preparation_receipt( + State(state): State, + Path((id, activity)): Path<(String, String)>, +) -> Result> { + Ok(Json(PreparationReceiptData { + data: state + .runtime + .snapshot_preparation_receipt(&id, &activity) + .await?, })) } async fn snapshot_preparation_review( diff --git a/crates/cyber-server/src/runtime/snapshot_preparation.rs b/crates/cyber-server/src/runtime/snapshot_preparation.rs index f26a40515..9ae8879ec 100644 --- a/crates/cyber-server/src/runtime/snapshot_preparation.rs +++ b/crates/cyber-server/src/runtime/snapshot_preparation.rs @@ -530,12 +530,31 @@ impl Runtime { pub async fn snapshot_preparations( &self, session: &str, + ) -> Result, RuntimeError> { + self.snapshot_preparations_with_history(session, false) + .await + } + pub async fn snapshot_preparations_with_history( + &self, + session: &str, + include_cancelled: bool, ) -> Result, RuntimeError> { self.session_info(session).await?; + let created = self + .inner + .store + .read_events(session, -1, 1)? + .events + .first() + .ok_or_else(|| { + RuntimeError::Conflict("Preparation Session creation is missing".into()) + })? + .id + .clone(); let session = session.to_owned(); self.inner.store.read(move |db| { - let mut query = db.prepare("SELECT aggregate_id,data FROM event e WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND json_extract(data,'$.status')<>'settled' AND COALESCE(json_extract(data,'$.snapshot_source_admitted'),0)=0 AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) ORDER BY aggregate_id LIMIT 4097")?; - let rows = query.query_map([session], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))?; + let mut query = db.prepare("SELECT aggregate_id,data FROM event e WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_worker')=1 AND ((json_extract(data,'$.status')<>'settled' AND COALESCE(json_extract(data,'$.snapshot_source_admitted'),0)=0) OR (?2 AND json_extract(data,'$.status')='settled' AND json_extract(data,'$.phase')='snapshot_preparation_cancelled' AND json_extract(data,'$.snapshot_preparation_receipt.lifetime.owner.session_created_id')=?3)) AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) ORDER BY aggregate_id LIMIT 4097")?; + let rows = query.query_map(rusqlite::params![session, include_cancelled, created], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))?; let mut result = Vec::new(); for row in rows { let (activity_id, data) = row?; @@ -756,6 +775,49 @@ impl Runtime { cancelled.disarm(); result } + /// Discover an exact committed cancellation; never infer safety from absence. + pub async fn snapshot_preparation_receipt( + &self, + session: &str, + activity: &str, + ) -> Result, RuntimeError> { + self.session_info(session).await?; + let source = session.to_owned(); + let owner = activity.to_owned(); + let request = self + .inner + .store + .read(move |db| { + let record: Option<(String, String)> = db.query_row( + "SELECT type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [owner], |row| Ok((row.get(0)?, row.get(1)?)), + ).optional()?; + let Some((kind, data)) = record else { + return Ok(None); + }; + let record: activity::Record = serde_json::from_str(&data) + .map_err(|_| invalid("Invalid preparation record"))?; + if kind != activity::CHANGED + || !record.snapshot_worker + || record.binary_document.is_some() + || record.session_id != source + { + return Err(invalid("Preparation discovery belongs to another owner")); + } + Ok(record + .snapshot_preparation_receipt + .map(|receipt| receipt.request_id)) + }) + .map_err(preparation_error)?; + match request { + Some(request) => { + self.snapshot_preparation_lookup(session, activity, &request) + .await + } + None => Ok(None), + } + } + pub async fn snapshot_preparation_lookup( &self, session: &str, diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index ed4401fc1..2691788eb 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -1253,6 +1253,49 @@ async fn cancelling_an_unstarted_resolution_keeps_its_original_failed_restore_fe ) .await .unwrap(); + assert!( + runtime + .snapshot_preparations(&session) + .await + .unwrap() + .iter() + .all(|candidate| candidate.activity_id != id) + ); + let history = runtime + .snapshot_preparations_with_history(&session, true) + .await + .unwrap(); + assert!( + history + .iter() + .any(|candidate| candidate.activity_id == id && candidate.status == "settled") + ); + let receipt = runtime + .snapshot_preparation_receipt(&session, &id) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.request_id, "cancel-resolution"); + assert_eq!( + runtime + .snapshot_preparation_lookup(&session, &id, &receipt.request_id) + .await + .unwrap(), + Some(receipt) + ); + assert!( + runtime + .snapshot_preparation_receipt(&session, "op_missing") + .await + .unwrap() + .is_none() + ); + assert!( + runtime + .snapshot_preparation_receipt("ses_foreign", &id) + .await + .is_err() + ); let discovered = runtime .snapshot_recovery_requests(&session, &original.binding.journal_id) .await diff --git a/crates/cyber-server/tests/http.rs b/crates/cyber-server/tests/http.rs index 5d098f76d..03b710685 100644 --- a/crates/cyber-server/tests/http.rs +++ b/crates/cyber-server/tests/http.rs @@ -1206,6 +1206,20 @@ fn openapi_document_is_current() { ); } +#[test] +fn preparation_discovery_schema_requires_the_nullable_data_field() { + let document = http::openapi::document("1"); + let schema = &document["components"]["schemas"]["PreparationReceiptData"]; + assert_eq!(schema["required"], json!(["data"])); + assert!( + schema["properties"]["data"]["anyOf"] + .as_array() + .unwrap() + .iter() + .any(|variant| variant["type"] == "null") + ); +} + #[tokio::test] async fn websocket_json_rpc_calls_operations_and_streams_events() { use futures::{SinkExt, StreamExt}; diff --git a/crates/cyber-tui/src/preparation_recovery.rs b/crates/cyber-tui/src/preparation_recovery.rs new file mode 100644 index 000000000..adc6f9edc --- /dev/null +++ b/crates/cyber-tui/src/preparation_recovery.rs @@ -0,0 +1,239 @@ +//! Pre-effect cancellation retains its exact request before sending any mutation. +use crate::{ + app::{Action, App}, + recovery, rewind, +}; +use crossterm::event::KeyCode; +use cyber_server::runtime::{ + SnapshotLifetimeBinding, SnapshotOrigin, SnapshotPreparation as Candidate, + SnapshotPreparationConfirmation as Confirmation, SnapshotPreparationReceipt as Receipt, + SnapshotPreparationReview as Review, +}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum Phase { + #[default] + Candidates, + Review, + Outcome, +} +#[derive(Debug, Default)] +pub struct View { + pub phase: Phase, + pub candidates: Vec, + pub selected: usize, + pub activity: Option, + pub review: Option, + pub request: Option, + pub receipt: Option, + pub reconciled: bool, +} +#[derive(Debug, Clone, PartialEq)] +pub enum Command { + List, + Discover(String), + Review(String), + Confirm(String, Confirmation), + Lookup(String, Confirmation), + Acknowledge(String, Confirmation), +} +#[derive(Debug, Clone)] +pub enum Reply { + Candidates(Vec), + Discovered(Option>), + Review(Box), + Receipt(Box), +} + +impl App { + pub(crate) fn preparation_request(&mut self, command: Command) -> Vec { + self.rewind.recovery.preparation.review = None; + self.rewind.recovery.preparation.receipt = None; + self.rewind.recovery.preparation.reconciled = false; + self.rewind.recovery.phase = recovery::Phase::Preparation; + self.rewind.scroll = 0; + self.recovery_request(recovery::Command::Preparation(command)) + } + pub(crate) fn preparation_key(&mut self, key: KeyCode) -> Vec { + if key == KeyCode::Char('j') { + return self.recovery_request(recovery::Command::List); + } + let view = &mut self.rewind.recovery.preparation; + if view.phase == Phase::Candidates { + return self.preparation_candidates_key(key); + } + let Some(activity) = view.activity.clone() else { + return vec![]; + }; + match (view.phase, key) { + (Phase::Review, KeyCode::Enter) if !self.session.running => { + let Some(review) = view.review.take() else { + return vec![]; + }; + let confirmation = Confirmation { + confirmation: review.confirmation, + request_id: cyber_core::ids::new_id("preparation"), + }; + view.request = Some(confirmation.clone()); + view.phase = Phase::Outcome; + self.preparation_request(Command::Confirm(activity, confirmation)) + } + (Phase::Review, KeyCode::Char('r')) => { + self.preparation_request(Command::Discover(activity)) + } + (Phase::Outcome, KeyCode::Char('r' | 'a')) => { + let Some(request) = view.request.clone() else { + return vec![]; + }; + if key == KeyCode::Char('a') && view.receipt.is_some() { + self.preparation_request(Command::Acknowledge(activity, request)) + } else if key == KeyCode::Char('r') { + self.preparation_request(Command::Lookup(activity, request)) + } else { + vec![] + } + } + _ => vec![], + } + } + fn preparation_candidates_key(&mut self, key: KeyCode) -> Vec { + let view = &mut self.rewind.recovery.preparation; + match key { + KeyCode::Up => view.selected = view.selected.saturating_sub(1), + KeyCode::Down => { + view.selected = (view.selected + 1).min(view.candidates.len().saturating_sub(1)) + } + KeyCode::Char('r') => return self.preparation_request(Command::List), + KeyCode::Enter => { + let Some(candidate) = view.candidates.get(view.selected) else { + return vec![]; + }; + let activity = candidate.activity_id.clone(); + view.activity = Some(activity.clone()); + view.request = None; + return self.preparation_request(Command::Discover(activity)); + } + _ => {} + } + vec![] + } +} + +pub(crate) fn apply( + app: &mut App, + request: &rewind::Request, + result: Result, +) -> Vec { + app.rewind.recovery.busy = false; + let rewind::Command::Recovery(recovery::Command::Preparation(command)) = &request.command + else { + return vec![]; + }; + let result = match result { + Ok(rewind::Reply::Recovery(reply)) => match *reply { + recovery::Reply::Preparation(reply) => accept( + &mut app.rewind.recovery.preparation, + request, + command, + *reply, + ), + _ => Err("Unexpected preparation response".into()), + }, + Err(error) => Err(error), + _ => Err("Unexpected preparation response".into()), + }; + match result { + Ok(Some(activity)) => app.preparation_request(Command::Review(activity)), + Ok(None) => vec![], + Err(error) => { + app.rewind.recovery.preparation.review = None; + app.rewind.recovery.preparation.receipt = None; + app.rewind.error = Some(error); + vec![] + } + } +} +fn accept( + view: &mut View, + request: &rewind::Request, + command: &Command, + reply: Reply, +) -> Result, String> { + match (command, reply) { + (Command::List, Reply::Candidates(candidates)) if candidates.len() <= 4096 => { + *view = View { candidates, ..Default::default() }; + } + (Command::Discover(activity), Reply::Discovered(None)) => return Ok(Some(activity.clone())), + (Command::Discover(activity), Reply::Discovered(Some(receipt))) if valid_receipt(&receipt, request, activity) => { + view.request = Some(Confirmation { confirmation: receipt.confirmation.clone(), request_id: receipt.request_id.clone() }); + view.receipt = Some(*receipt); + view.phase = Phase::Outcome; + } + (Command::Review(activity), Reply::Review(review)) if valid_review(&review, request, activity) => { + view.review = Some(*review); + view.phase = Phase::Review; + } + (Command::Confirm(activity, expected) | Command::Lookup(activity, expected) | Command::Acknowledge(activity, expected), Reply::Receipt(receipt)) + if valid_receipt(&receipt, request, activity) && receipt.request_id == expected.request_id && receipt.confirmation == expected.confirmation => { + view.receipt = Some(*receipt); + view.reconciled = matches!(command, Command::Acknowledge(..)); + view.phase = Phase::Outcome; + } + _ => return Err("Preparation reply lacks matching ownership proof; retain the request and refresh without replay".into()), + } + Ok(None) +} +fn fingerprint(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} +fn owner_matches( + lifetime: &SnapshotLifetimeBinding, + origin: Option<&SnapshotOrigin>, + request: &rewind::Request, + activity: &str, +) -> bool { + lifetime.owner.activity_id == activity + && lifetime.owner.session_id == request.session + && lifetime.owner.directory == std::path::Path::new(&request.directory) + && origin.is_none_or(|origin| { + origin.version == 1 + && lifetime.version == 2 + && origin.session_id == lifetime.owner.session_id + && origin.activity_id == lifetime.owner.activity_id + && origin.session_created_id == lifetime.owner.session_created_id + && origin.worktree_id == lifetime.owner.worktree_id + && origin.binding.directory == lifetime.owner.directory + && origin.binding.root_identity == lifetime.root_identity + }) +} +fn valid_review(review: &Review, request: &rewind::Request, activity: &str) -> bool { + review.activity_id == activity + && review.lifetime.version == 2 + && review.lifetime.source_phase.is_some() + && fingerprint(&review.confirmation) + && owner_matches(&review.lifetime, review.origin.as_ref(), request, activity) +} +fn valid_receipt(receipt: &Receipt, request: &rewind::Request, activity: &str) -> bool { + receipt.activity_id == activity + && fingerprint(&receipt.confirmation) + && !receipt.request_id.is_empty() + && receipt.request_id.len() <= 256 + && !receipt.request_id.contains('\0') + && receipt.source_admitted != Some(true) + && (receipt.origin.is_none() || receipt.source_admitted == Some(false)) + && owner_matches( + &receipt.lifetime, + receipt.origin.as_ref(), + request, + activity, + ) +} + +#[path = "preparation_recovery_view.rs"] +mod view; +pub use view::lines; +#[path = "preparation_recovery_client.rs"] +pub mod client; +#[cfg(test)] +#[path = "preparation_recovery_tests.rs"] +mod tests; diff --git a/crates/cyber-tui/src/preparation_recovery_client.rs b/crates/cyber-tui/src/preparation_recovery_client.rs new file mode 100644 index 000000000..ef583c1f4 --- /dev/null +++ b/crates/cyber-tui/src/preparation_recovery_client.rs @@ -0,0 +1,62 @@ +use super::{Command, Reply}; +use crate::{ + recovery::client::{data, segment}, + rewind::Request, +}; +use cyber_client::Client; +use serde_json::json; + +pub async fn perform( + client: &Client, + request: &Request, + command: &Command, +) -> Result { + let base = format!( + "/sessions/{}/rewind/preparations", + segment(&request.session) + ); + match command { + Command::List => { + let reply = client + .get(&format!("{base}?include_cancelled=true")) + .await + .map_err(|error| error.to_string())?; + Ok(Reply::Candidates(data(reply)?)) + } + Command::Discover(activity) => { + let reply = client + .get(&format!("{base}/{}/receipt", segment(activity))) + .await + .map_err(|error| error.to_string())?; + Ok(Reply::Discovered(data(reply)?)) + } + Command::Review(activity) => { + let reply = client + .post(&format!("{base}/{}/review", segment(activity)), json!({})) + .await + .map_err(|error| error.to_string())?; + Ok(Reply::Review(Box::new(data(reply)?))) + } + _ => receipt(client, &base, command).await, + } +} +async fn receipt(client: &Client, base: &str, command: &Command) -> Result { + let reply = match command { + Command::Confirm(activity, confirmation) => { + client.post(&format!("{base}/{}/confirm", segment(activity)), json!(confirmation)).await + } + Command::Lookup(activity, confirmation) | Command::Acknowledge(activity, confirmation) => { + if matches!(confirmation.request_id.as_str(), "." | "..") { + return Err("Retained request ID cannot be addressed by this path API; evidence is preserved".into()); + } + let path = format!("{base}/{}/requests/{}", segment(activity), segment(&confirmation.request_id)); + if matches!(command, Command::Acknowledge(..)) { + client.post(&format!("{path}/acknowledge"), json!({})).await + } else { + client.get(&path).await + } + } + _ => return Err("Expected an exact retained preparation request".into()), + }.map_err(|error| format!("{error}. Retain the original request; R looks it up without replay"))?; + Ok(Reply::Receipt(Box::new(data(reply)?))) +} diff --git a/crates/cyber-tui/src/preparation_recovery_tests.rs b/crates/cyber-tui/src/preparation_recovery_tests.rs new file mode 100644 index 000000000..dec10f169 --- /dev/null +++ b/crates/cyber-tui/src/preparation_recovery_tests.rs @@ -0,0 +1,255 @@ +use super::*; +use crate::{app::Action, model::Session}; +use serde_json::json; +fn app() -> App { + let mut app = App::new( + Session { + id: "ses_one".into(), + directory: "/project".into(), + ..Default::default() + }, + vec![], + "cyber", + ); + app.composer.set_text("owner draft"); + app.attachments.chips.push(crate::attachments::Chip { + kind: crate::attachments::Kind::Text, + number: 1, + payload: crate::attachments::Payload::Text { + text: "attachment owner data".into(), + }, + }); + app +} +fn request(actions: Vec) -> rewind::Request { + let [Action::Rewind(request)] = actions.as_slice() else { + panic!("one guarded owner request") + }; + request.clone() +} +fn deliver(app: &mut App, request: rewind::Request, reply: Reply) -> Vec { + rewind::apply( + app, + &mut Default::default(), + request, + Ok(rewind::Reply::Recovery(Box::new( + recovery::Reply::Preparation(Box::new(reply)), + ))), + ) +} +fn review() -> Review { + serde_json::from_value(json!({"activity_id":"op_one","lifetime":{"version":2,"owner":{"activity_id":"op_one","session_id":"ses_one","session_created_id":"created","directory":"/project","worktree_id":null},"root_identity":"root","namespace_identity":"namespace","directory_identity":"directory","file_identity":"file","source_phase":"phase"},"native_sequence":0,"session_sequence":1,"confirmation":"a".repeat(64),"warning":"independent uncertainty remains","origin":null,"checkout_resources_digest":"d".repeat(64)})).unwrap() +} +fn receipt(id: &str) -> Receipt { + let review = review(); + Receipt { + activity_id: review.activity_id, + request_id: id.into(), + confirmation: review.confirmation, + lifetime: review.lifetime, + native_sequence: 0, + session_sequence: 1, + origin: None, + source_admitted: Some(false), + checkout_resources_digest: review.checkout_resources_digest, + } +} +fn open(app: &mut App) { + let req = request(app.open_snapshot_recovery()); + rewind::apply( + app, + &mut Default::default(), + req, + Ok(rewind::Reply::Recovery(Box::new( + recovery::Reply::Journals(vec![]), + ))), + ); + let req = request(app.recovery_key(KeyCode::Char('p'))); + deliver( + app, + req, + Reply::Candidates(vec![Candidate { + activity_id: "op_one".into(), + status: "unknown".into(), + lifetime_available: true, + }]), + ); +} +#[test] +fn cancellation_retains_key_before_admission_and_never_replays_after_reply_loss() { + let mut app = app(); + open(&mut app); + let req = request(app.recovery_key(KeyCode::Enter)); + let actions = deliver(&mut app, req, Reply::Discovered(None)); + let req = request(actions); + assert!(matches!( + req.command, + rewind::Command::Recovery(recovery::Command::Preparation(Command::Review(_))) + )); + deliver(&mut app, req, Reply::Review(Box::new(review()))); + let confirm = request(app.recovery_key(KeyCode::Enter)); + let retained = app.rewind.recovery.preparation.request.clone().unwrap(); + assert!( + matches!(&confirm.command, rewind::Command::Recovery(recovery::Command::Preparation(Command::Confirm(_, expected))) if expected == &retained) + ); + assert!(app.rewind.recovery.preparation.review.is_none()); + assert!(app.recovery_key(KeyCode::Enter).is_empty()); + rewind::apply( + &mut app, + &mut Default::default(), + confirm, + Err("lost reply".into()), + ); + assert!(app.recovery_key(KeyCode::Enter).is_empty()); + lookup_and_reconcile(&mut app, &retained); + assert_eq!(app.composer.text(), "owner draft"); + assert!(app.items.is_empty()); + assert_eq!(app.attachments.chips.len(), 1); + assert_eq!( + app.attachments.chips[0].payload, + crate::attachments::Payload::Text { + text: "attachment owner data".into() + } + ); +} +fn lookup_and_reconcile(app: &mut App, retained: &Confirmation) { + let lookup = request(app.recovery_key(KeyCode::Char('r'))); + assert!( + matches!(&lookup.command, rewind::Command::Recovery(recovery::Command::Preparation(Command::Lookup(_, expected))) if expected == retained) + ); + deliver( + app, + lookup, + Reply::Receipt(Box::new(receipt(&retained.request_id))), + ); + assert!(!app.rewind.recovery.preparation.reconciled); + let ack = request(app.recovery_key(KeyCode::Char('a'))); + deliver( + app, + ack, + Reply::Receipt(Box::new(receipt(&retained.request_id))), + ); + assert!(app.rewind.recovery.preparation.reconciled); +} + +#[test] +fn fresh_terminal_discovers_committed_cancellation_and_only_reconciles_explicitly() { + let mut app = app(); + open(&mut app); + let req = request(app.recovery_key(KeyCode::Enter)); + assert!( + deliver( + &mut app, + req, + Reply::Discovered(Some(Box::new(receipt("original / #?%")))) + ) + .is_empty() + ); + assert_eq!(app.rewind.recovery.preparation.phase, Phase::Outcome); + assert_eq!( + app.rewind + .recovery + .preparation + .request + .as_ref() + .unwrap() + .request_id, + "original / #?%" + ); + assert!(!app.rewind.recovery.preparation.reconciled); + assert!(app.recovery_key(KeyCode::Enter).is_empty()); + assert!( + lines(&app) + .iter() + .any(|line| line.to_string().contains("not established")) + ); + assert_eq!(app.composer.text(), "owner draft"); +} +#[test] +fn missing_phase_foreign_location_and_stale_reply_never_grant_confirmation() { + let mut app = app(); + open(&mut app); + app.rewind.recovery.preparation.phase = Phase::Review; + for mode in 0..3 { + let req = request(app.preparation_request(Command::Review("op_one".into()))); + let mut invalid = review(); + match mode { + 0 => invalid.lifetime.source_phase = None, + 1 => invalid.lifetime.owner.directory = "/foreign".into(), + _ => invalid.confirmation = "bad".into(), + } + deliver(&mut app, req, Reply::Review(Box::new(invalid))); + assert!(app.rewind.recovery.preparation.review.is_none()); + assert!(app.recovery_key(KeyCode::Enter).is_empty()); + } + let stale = request(app.preparation_request(Command::Review("op_one".into()))); + let current = request(app.preparation_request(Command::List)); + deliver(&mut app, stale, Reply::Review(Box::new(review()))); + assert!(app.rewind.recovery.busy); + assert!(app.rewind.recovery.preparation.review.is_none()); + deliver(&mut app, current, Reply::Candidates(vec![])); + assert_eq!(app.composer.text(), "owner draft"); +} + +#[tokio::test] +async fn actual_transport_reads_history_and_absence_still_requires_positive_review() { + let server = crate::screens::Server::start().await; + let created = server + .client + .post("/sessions", json!({"model":"test/main"})) + .await + .unwrap(); + let id = created["data"]["id"].as_str().unwrap(); + let mut app = server.load(id).await; + app.composer.set_text("transport owner draft"); + let mut actions = app.open_snapshot_recovery(); + while let Some(action) = actions.pop() { + let reply = crate::perform::perform_owned(&server.client, &app.session, action, None).await; + actions.extend(crate::runner::apply( + &mut app, + reply, + &mut Default::default(), + )); + } + let list = app.recovery_key(KeyCode::Char('p')).pop().unwrap(); + let reply = crate::perform::perform_owned(&server.client, &app.session, list, None).await; + crate::runner::apply(&mut app, reply, &mut Default::default()); + assert!(app.rewind.recovery.preparation.candidates.is_empty()); + let absent = request(app.preparation_request(Command::Discover("op_missing".into()))); + let reply = recovery::perform( + &server.client, + &absent, + match &absent.command { + rewind::Command::Recovery(command) => command, + _ => unreachable!(), + }, + ) + .await + .unwrap(); + let recovery::Reply::Preparation(reply) = reply else { + panic!("preparation reply") + }; + let review = request(deliver(&mut app, absent, *reply)); + let command = match &review.command { + rewind::Command::Recovery(command) => command, + _ => unreachable!(), + }; + let error = recovery::perform(&server.client, &review, command) + .await + .unwrap_err(); + rewind::apply(&mut app, &mut Default::default(), review, Err(error)); + assert!(app.rewind.recovery.preparation.review.is_none()); + assert!(app.rewind.recovery.preparation.request.is_none()); + assert_eq!(app.composer.text(), "transport owner draft"); + assert!(server.model.requests().is_empty()); + assert!( + server + .client + .get(&format!( + "/sessions/{id}/rewind/preparations/op_missing/receipt" + )) + .await + .unwrap()["data"] + .is_null() + ); +} diff --git a/crates/cyber-tui/src/preparation_recovery_view.rs b/crates/cyber-tui/src/preparation_recovery_view.rs new file mode 100644 index 000000000..46cee2f3d --- /dev/null +++ b/crates/cyber-tui/src/preparation_recovery_view.rs @@ -0,0 +1,99 @@ +use super::{Phase, View}; +use crate::app::App; +use ratatui::text::Line; + +pub fn lines(app: &App) -> Vec> { + let view = &app.rewind.recovery.preparation; + let mut lines = vec![ + Line::from("PREPARATION RECOVERY · J journals · Esc closes"), + Line::from( + "Files, conversation and draft are preserved. Independent uncertainty remains protected.", + ), + ]; + if app.rewind.recovery.busy { + lines.push(Line::from( + "Waiting for owner server; closing does not undo admission.", + )); + } + if let Some(request) = &view.request { + lines.push(Line::from(format!( + "Retained request: {:?}", + request.request_id + ))); + } + match view.phase { + Phase::Candidates => candidates(view, &mut lines), + Phase::Review => review(view, &mut lines), + Phase::Outcome => outcome(view, &mut lines), + } + lines +} +fn candidates(view: &View, lines: &mut Vec>) { + lines.push(Line::from( + "↑/↓ select · Enter discovers a committed receipt before review · R reloads", + )); + if view.candidates.is_empty() { + lines.push(Line::from( + "No retained preparation candidates. No uncertainty is cleared.", + )); + } + for (index, candidate) in view + .candidates + .iter() + .enumerate() + .skip(view.selected.saturating_sub(4)) + .take(9) + { + lines.push(Line::from(format!( + "{} {:?} · status {:?} · lifetime evidence={}", + if index == view.selected { ">" } else { " " }, + candidate.activity_id, + candidate.status, + candidate.lifetime_available + ))); + } +} +fn review(view: &View, lines: &mut Vec>) { + let Some(review) = &view.review else { + return; + }; + lines.push(Line::from(format!( + "Review {:?} · Location {:?}", + review.activity_id, review.lifetime.owner.directory + ))); + lines.push(Line::from(format!( + "Lifetime v{} · native sequence {} · Session sequence {} · origin published={}", + review.lifetime.version, + review.native_sequence, + review.session_sequence, + review.origin.is_some() + ))); + lines.push(Line::from(format!( + "Acquired checkout digest: {:?}", + review.checkout_resources_digest + ))); + lines.push(Line::from(crate::messaging::clean(&review.warning))); + lines.push(Line::from( + "Enter cancels only this reviewed pre-effect preparation · R refreshes", + )); +} +fn outcome(view: &View, lines: &mut Vec>) { + let Some(receipt) = &view.receipt else { + lines.push(Line::from( + "Outcome unknown. R looks up the retained request without repeating cancellation.", + )); + return; + }; + lines.push(Line::from(format!( + "Committed cancellation {:?} · Location {:?}", + receipt.activity_id, receipt.lifetime.owner.directory + ))); + lines.push(Line::from(format!( + "Lifetime v{} · origin published={} · acquired checkout digest: {:?}", + receipt.lifetime.version, + receipt.origin.is_some(), + receipt.checkout_resources_digest + ))); + lines.push(Line::from(if view.reconciled { "Explicit acknowledgement reconciliation completed for this receipt." } else { "Receipt proves durable cancellation. Native acknowledgement is not established by this response alone." })); + lines.push(Line::from("R looks up the same request · A reconciles native metadata · Enter never repeats cancellation")); +} diff --git a/crates/cyber-tui/src/recovery.rs b/crates/cyber-tui/src/recovery.rs index 46ca66146..8c4488c05 100644 --- a/crates/cyber-tui/src/recovery.rs +++ b/crates/cyber-tui/src/recovery.rs @@ -12,6 +12,8 @@ use cyber_server::runtime::{ }; #[path = "recovery_client.rs"] mod client; +#[path = "preparation_recovery.rs"] +mod preparation; pub use client::perform; #[path = "recovery_view.rs"] mod view; @@ -21,6 +23,7 @@ pub use view::lines; pub enum Phase { #[default] Journals, + Preparation, Requests, Review, Outcome, @@ -29,6 +32,7 @@ pub enum Phase { pub struct View { pub phase: Phase, pub busy: bool, + pub preparation: preparation::View, pub discovered: bool, pub journals: Vec, pub requests: Vec, @@ -40,6 +44,7 @@ pub struct View { } #[derive(Debug, Clone, PartialEq)] pub enum Command { + Preparation(preparation::Command), List, Discover(String), Review(Selection), @@ -49,6 +54,7 @@ pub enum Command { } #[derive(Debug, Clone)] pub enum Reply { + Preparation(Box), Journals(Vec), Requests(Vec), Review(Box), @@ -63,7 +69,7 @@ impl App { self.rewind.scroll = 0; self.recovery_request(Command::List) } - fn recovery_request(&mut self, command: Command) -> Vec { + pub(crate) fn recovery_request(&mut self, command: Command) -> Vec { match &command { Command::Discover(_) => self.rewind.recovery.discovered = false, Command::Review(_) => self.rewind.recovery.review = None, @@ -92,7 +98,11 @@ impl App { if self.rewind.recovery.busy { return vec![]; } + if key == KeyCode::Char('p') { + return self.preparation_request(preparation::Command::List); + } match self.rewind.recovery.phase { + Phase::Preparation => self.preparation_key(key), Phase::Journals => self.recovery_journal_key(key), Phase::Requests => self.recovery_requests_key(key), Phase::Review => self.recovery_review_key(key), @@ -258,6 +268,12 @@ pub(crate) fn apply( request: &rewind::Request, result: Result, ) -> Vec { + if matches!( + request.command, + rewind::Command::Recovery(Command::Preparation(_)) + ) { + return preparation::apply(app, request, result); + } app.rewind.recovery.busy = false; match result { Err(error) => { diff --git a/crates/cyber-tui/src/recovery_client.rs b/crates/cyber-tui/src/recovery_client.rs index 3bed3b720..d62d2f0d4 100644 --- a/crates/cyber-tui/src/recovery_client.rs +++ b/crates/cyber-tui/src/recovery_client.rs @@ -4,7 +4,7 @@ use cyber_client::Client; use serde::de::DeserializeOwned; use serde_json::{Value, json}; -fn segment(value: &str) -> String { +pub(super) fn segment(value: &str) -> String { value .bytes() .map(|byte| { @@ -16,7 +16,7 @@ fn segment(value: &str) -> String { }) .collect() } -fn data(reply: Value) -> Result { +pub(super) fn data(reply: Value) -> Result { serde_json::from_value(reply["data"].clone()) .map_err(|error| format!("Invalid recovery reply: {error}")) } @@ -36,6 +36,11 @@ pub async fn perform( } let base = format!("/sessions/{}/rewind/recovery", segment(&request.session)); match command { + Command::Preparation(command) => { + super::preparation::client::perform(client, request, command) + .await + .map(|reply| Reply::Preparation(Box::new(reply))) + } Command::List => { let journals: Vec = data(client.get(&base).await.map_err(|error| error.to_string())?)?; diff --git a/crates/cyber-tui/src/recovery_view.rs b/crates/cyber-tui/src/recovery_view.rs index d9e0223cc..04fea5bff 100644 --- a/crates/cyber-tui/src/recovery_view.rs +++ b/crates/cyber-tui/src/recovery_view.rs @@ -5,6 +5,9 @@ use ratatui::text::Line; pub fn lines(app: &App) -> Vec> { let view = &app.rewind.recovery; + if view.phase == Phase::Preparation { + return super::preparation::lines(app); + } let mut lines = vec![ Line::from("SNAPSHOT RECOVERY · composer and conversation stay unchanged"), Line::from("Esc closes · PgUp/PgDn scroll · external effects remain retained"), @@ -21,6 +24,7 @@ pub fn lines(app: &App) -> Vec> { ))); } match view.phase { + Phase::Preparation => unreachable!(), Phase::Journals => journals(view, &mut lines), Phase::Requests => requests(view, &mut lines), Phase::Review => review(view, &mut lines), @@ -30,10 +34,12 @@ pub fn lines(app: &App) -> Vec> { } fn journals(view: &View, lines: &mut Vec>) { lines.push(Line::from( - "↑/↓ select journal · Enter discovers retained requests · R reloads", + "↑/↓ select journal · Enter discovers retained requests · R reloads · P preparations", )); if view.journals.is_empty() { - lines.push(Line::from("No retained journals. Pre-effect preparations may still need their preparation API; no fence is cleared.")); + lines.push(Line::from( + "No retained journals. P opens preparation recovery; no fence is cleared.", + )); } for (index, journal) in view .journals diff --git a/docs/rewind.md b/docs/rewind.md index b14e9ad56..db254e5d6 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -403,9 +403,8 @@ state with exit 0 because the read succeeded. Both commands support `--format js These commands expose the implemented managed file-copy recovery path on admitted Linux/macOS hosts. Ordinary managed Git source recovery remains disabled pending -its remaining compatibility and runtime acceptance gates. Preparation cancellation -is still available through its existing API; typed preparation/TUI recovery, -reference-aware reclamation and complete native/platform acceptance remain open. +its remaining compatibility and runtime acceptance gates. Reference-aware +reclamation and complete native/platform acceptance remain open. ## Recovery in the terminal panel @@ -433,8 +432,24 @@ Select a journal with Up/Down, then Enter to discover its retained requests. Recovery preserves the composer, attachments and conversation. Replies from an old Session, Location or UI generation are ignored. Missing legacy proof never enables confirmation, and failed request discovery cannot license a fresh attempt. -Preparation cancellation still uses the existing preparation API; its typed panel -and the remaining managed Git/cleanup/platform acceptance work are unfinished. +Press **P** in recovery to inspect pre-effect preparations, including committed +cancellations. Select a candidate and press Enter: the panel first discovers an +existing cancellation receipt; otherwise it requests a fresh pre-effect review. +Only a matching current review offers Enter to cancel. The panel retains the +request identity before sending confirmation. After an ambiguous reply, **R** +looks up that same request; Enter cannot repeat cancellation. **A** explicitly +reconciles a committed receipt's native metadata. **J** returns to journals. +A receipt alone does not establish observed native acknowledgement. Preparation +cancellation preserves independent restore, process and checkout uncertainty. + +The panel uses the opt-in read +`GET /api/v1/sessions/{id}/rewind/preparations?include_cancelled=true` and exact +receipt discovery at `/rewind/preparations/{activity_id}/receipt`. Default listing +continues to return pending candidates. History is scoped to the current Session +creation; discovery validates the committed receipt pair and does not touch +source files or perform native reconciliation. An absent receipt grants no +cancellation authority: native proof must still pass review and confirmation. +Managed Git, cleanup and complete crash/platform acceptance remain unfinished. `GET /api/v1/sessions/{id}/rewind/recovery/requests?journal_id=restore_example` provides the read-only discovery used by the panel. It requires an exact journal, diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index dcdcd2830..4ce77c184 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3024,3 +3024,57 @@ workspace, Linux/Windows native or source-bound CI checks ran for this command-o continuation. Preparation TUI, partial constructors, managed Git source integration, reference-aware reclamation and broader acceptance remain open; no parent task, issue or Git availability guard is promoted. No PR/push is made by this local work. + +## Preparation recovery in the terminal and committed receipt discovery (2026-11-07) + +The rewind recovery panel now offers P for preparation candidates, including an +opt-in history of committed cancellations scoped to the current Session creation. +Default API listing remains pending-only. Candidate Enter discovers the original +receipt using a new read-only route before requesting any pre-effect review. +Absence itself grants no authority. Review/confirmation still require exact exited +native worker proof; no source-admission or legacy-fence policy changes. + +The nested terminal controller uses the existing rewind Session/Location/UI +generation guard. It validates activity/lifetime/origin and confirmation scope, +retains the stable cancellation identity before POST, consumes the review and +makes outcome Enter inert. R looks up that exact request; A explicitly reconciles +native metadata only after a committed receipt. A receipt alone is never displayed +as observed native acknowledgement. Independent restore/process/checkout +uncertainty, draft, attachment payloads and conversation are preserved. J returns +to journal recovery. The HTTP transport stays separate from controller/rendering. + +The new receipt endpoint discovers the retained activity request key and delegates +to existing exact paired-receipt validation. Its always-present nullable data field +is explicit in OpenAPI so the generated SDK returns the receipt or null. Discovery +never calls native reconciliation or accesses source files. The existing native +CLI crash fixtures now verify pending/history separation and receipt discovery +after a real service restart, plus refusal for an existing foreign Session. + +Code-review skill inspection covered all changed controller/transport/rendering, +HTTP/Store queries, generated artifacts, tests and docs. Scope guards, busy gates, +consumed confirmation, ambiguous-reply behavior, native acknowledgement separation +and explicit null handling were checked. No unresolved critical/major finding was +identified in this increment. Rust complexity uses native Clippy because the +cognitive-complexity skill dispatcher does not support Rust; 18/15 findings in +the controller test and native CLI fixture were resolved by separating +lookup/reconciliation and history checks, without lint allowances. The shared managed Git guard remains disabled. Native preparation +recovery through an actual killed TUI process, full crash/platform cuts, partial +constructors, reclamation and remaining #93/#92 work remain open. + +Local verification for this increment: all 675 TUI library tests passed (3 ignored +cases: the separately driven pure-mode child fixture and two Linux protected-file +agent/collaboration cases avoiding native vault writes on macOS); the native +partial-journal interruption/reopen regression passed within that suite. All 80 +HTTP tests, the focused cancellation/history/independent-fence runtime regression, +and the two real CLI native pre-effect service-kill cuts passed. The final SDK +suite passed 144 tests with 2 Linux protected-file/native-vault platform skips; +typechecking, generated SDK consistency, all-target server/TUI/CLI Clippy, pinned +strict OpenSpec (117 items), and formatting/diff checks passed. Spec lint reported +0 errors/25 warnings, without unchanged-main attribution. Every Cargo +build/check/test/Clippy used CARGO_INCREMENTAL=0 and --locked. The nullable receipt +schema has a dedicated regression and the SDK test statically admits null. +C deprecation and oversized-unwind linker warnings were observed without origin +attribution. Full workspace, native preparation TUI process-kill, additional +Linux/macOS architectures, Windows and source-bound CI were not run here; this is +local evidence, not complete M7/M8 or accepted delivery. No PR/push is made by this +increment and no parent task or issue is closed. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 53a3671f0..9f0b055b7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -774,3 +774,16 @@ metadata through the existing route. Replies SHALL match the selected Session, activity and lifetime, and lookup SHALL NOT imply observed native acknowledgement. No missing journal, historical lifetime or completed filesystem work SHALL be promoted into cancellation authority by these clients. + +### Preparation recovery in a fresh terminal + +The terminal recovery panel SHALL expose preparation review/cancellation through +its existing pre-effect protocol. An opt-in candidate history SHALL include exact +committed cancellations for the current Session creation; default candidate listing +SHALL remain unchanged. Exact activity receipt discovery SHALL validate the existing +committed pair and SHALL NOT read source files or publish native acknowledgement. +The panel SHALL discover an existing receipt before reviewing another cancellation, +retain a request ID before confirmation, use lookup after reply loss, and keep +native reconciliation explicit. Stale/foreign replies and missing current proof +SHALL NOT enable cancellation; closing/navigation SHALL preserve drafts, attachments +and all independent uncertainty. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/tui/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/tui/spec.md index 510bb1931..e6e66fcd2 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/tui/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/tui/spec.md @@ -28,3 +28,12 @@ #### Scenario: Preserve unrelated files through repeated undo - **WHEN** several reviewed code stages are replayed and then cleared - **THEN** the original baseline and durable set of changed paths constrain clear, and unrelated owner files are not adopted into redo authority + +#### Scenario: Preparation cancellation discovery after reply loss +- GIVEN an exact pre-effect snapshot preparation or its committed cancellation remains retained for the current Session creation +- WHEN the owner opens preparation recovery from the rewind recovery panel +- THEN the terminal SHALL discover an existing cancellation receipt before offering a new native-proof review +- AND confirmation SHALL retain its stable request identity before admission and drop the consumed review +- AND a lost reply SHALL permit lookup of that original request without replaying cancellation +- AND native metadata reconciliation SHALL require an explicit action and preserve independent uncertainty +- AND stale Session, Location or UI generation responses SHALL NOT change the draft, attachments or confirmation authority diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 89cb45709..053c38cd2 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -502,3 +502,18 @@ including stale/foreign refusal and preservation of later owner symlinks/files. This advances the preparation CLI portion of M7; preparation TUI, partial construction, managed Git integration, safe reclamation and complete acceptance remain open. No parent task is closed. + +Preparation terminal continuation: rewind recovery now offers P for preparation +candidates and committed cancellation history, with J returning to journals. +Candidate selection discovers a committed receipt before native-proof review; +confirmation retains a fresh stable request before sending, consumes the review, +and permits lookup rather than Enter replay after reply loss. A is explicit +metadata reconciliation. Session/Location/generation guards preserve composer, +attachments and conversation. Opt-in history keeps default pending listing intact; +exact receipt discovery delegates to the validated original committed pair. +Controller and actual HTTP transport coverage accompanies native CLI service-kill +cuts before/after origin publication, including restarted history/discovery and +foreign-Session refusal. Actual native preparation cancellation driven by a killed +TUI process remains a separate acceptance gap. M7/M8 and their parent tasks stay +open alongside partial constructors, managed Git admission, safe reclamation and +complete compatibility/platform evidence. diff --git a/sdk/openapi.json b/sdk/openapi.json index db1e22e08..fdf4c7624 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -7733,23 +7733,38 @@ "lifetime_available" ] }, - "Data_SnapshotPreparationReview": { + "PreparationReceiptData": { "type": "object", "properties": { "data": { - "$ref": "#/components/schemas/SnapshotPreparationReview" + "anyOf": [ + { + "$ref": "#/components/schemas/SnapshotPreparationReceipt" + }, + { + "type": "null" + } + ] } }, + "description": "A discovery response always contains `data`, including an explicit absent receipt.", "required": [ "data" ] }, - "SnapshotPreparationReview": { + "SnapshotPreparationReceipt": { "type": "object", + "additionalProperties": false, "properties": { "activity_id": { "type": "string" }, + "request_id": { + "type": "string" + }, + "confirmation": { + "type": "string" + }, "lifetime": { "$ref": "#/components/schemas/SnapshotLifetimeBinding" }, @@ -7761,12 +7776,6 @@ "type": "integer", "format": "int64" }, - "confirmation": { - "type": "string" - }, - "warning": { - "type": "string" - }, "origin": { "anyOf": [ { @@ -7777,6 +7786,12 @@ } ] }, + "source_admitted": { + "type": [ + "boolean", + "null" + ] + }, "checkout_resources_digest": { "type": [ "string", @@ -7786,53 +7801,30 @@ }, "required": [ "activity_id", + "request_id", + "confirmation", "lifetime", "native_sequence", - "session_sequence", - "confirmation", - "warning" - ] - }, - "SnapshotPreparationConfirmation": { - "type": "object", - "additionalProperties": false, - "properties": { - "confirmation": { - "type": "string" - }, - "request_id": { - "type": "string" - } - }, - "required": [ - "confirmation", - "request_id" + "session_sequence" ] }, - "Data_SnapshotPreparationReceipt": { + "Data_SnapshotPreparationReview": { "type": "object", "properties": { "data": { - "$ref": "#/components/schemas/SnapshotPreparationReceipt" + "$ref": "#/components/schemas/SnapshotPreparationReview" } }, "required": [ "data" ] }, - "SnapshotPreparationReceipt": { + "SnapshotPreparationReview": { "type": "object", - "additionalProperties": false, "properties": { "activity_id": { "type": "string" }, - "request_id": { - "type": "string" - }, - "confirmation": { - "type": "string" - }, "lifetime": { "$ref": "#/components/schemas/SnapshotLifetimeBinding" }, @@ -7844,6 +7836,12 @@ "type": "integer", "format": "int64" }, + "confirmation": { + "type": "string" + }, + "warning": { + "type": "string" + }, "origin": { "anyOf": [ { @@ -7854,12 +7852,6 @@ } ] }, - "source_admitted": { - "type": [ - "boolean", - "null" - ] - }, "checkout_resources_digest": { "type": [ "string", @@ -7869,11 +7861,38 @@ }, "required": [ "activity_id", - "request_id", - "confirmation", "lifetime", "native_sequence", - "session_sequence" + "session_sequence", + "confirmation", + "warning" + ] + }, + "SnapshotPreparationConfirmation": { + "type": "object", + "additionalProperties": false, + "properties": { + "confirmation": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "confirmation", + "request_id" + ] + }, + "Data_SnapshotPreparationReceipt": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/SnapshotPreparationReceipt" + } + }, + "required": [ + "data" ] }, "RewindSelection": { @@ -25073,6 +25092,14 @@ "schema": { "type": "string" } + }, + { + "name": "include_cancelled", + "in": "query", + "required": false, + "schema": { + "type": "string" + } } ], "responses": { @@ -25099,6 +25126,55 @@ } } }, + "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/receipt": { + "get": { + "operationId": "v1.session.snapshotPreparationReceipt", + "summary": "Discover the exact committed preparation cancellation receipt without source access or native acknowledgement; absence is not pre-effect proof", + "tags": [ + "session" + ], + "parameters": [ + { + "name": "sessionID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "activityID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PreparationReceiptData" + } + } + } + }, + "default": { + "description": "Tagged error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/review": { "post": { "operationId": "v1.session.snapshotPreparationReview", diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index 1981ab56f..e66c4151a 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1571,6 +1571,12 @@ export interface Operations { body: undefined; response: S.SnapshotPreparationReceipt; }; + "v1.session.snapshotPreparationReceipt": { + path: { sessionID: string; activityID: string }; + query: Record; + body: undefined; + response: S.SnapshotPreparationReceipt | null; + }; "v1.session.snapshotPreparationReview": { path: { sessionID: string; activityID: string }; query: Record; @@ -1579,7 +1585,9 @@ export interface Operations { }; "v1.session.snapshotPreparations": { path: { sessionID: string }; - query: Record; + query: { + include_cancelled?: QueryValue; + }; body: undefined; response: S.SnapshotPreparation[]; }; @@ -4742,6 +4750,18 @@ export const operations = { unwrap: true, stream: false, }, + "v1.session.snapshotPreparationReceipt": { + tag: "session", + name: "snapshotPreparationReceipt", + method: "GET", + path: "/api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/receipt", + pathParams: ["sessionID", "activityID"], + query: [], + body: "none", + located: false, + unwrap: true, + stream: false, + }, "v1.session.snapshotPreparationReview": { tag: "session", name: "snapshotPreparationReview", @@ -4760,7 +4780,7 @@ export const operations = { method: "GET", path: "/api/v1/sessions/{sessionID}/rewind/preparations", pathParams: ["sessionID"], - query: [], + query: ["include_cancelled"], body: "none", located: false, unwrap: true, @@ -6261,12 +6281,15 @@ export function createGroups(caller: Caller) { /** Look up the original durable preparation cancellation pair without native access or replay (`GET /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/requests/{requestID}`) */ snapshotPreparationLookup: (sessionID: string, activityID: string, requestID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotPreparationLookup", { path: { sessionID, activityID, requestID } }, options), + /** Discover the exact committed preparation cancellation receipt without source access or native acknowledgement; absence is not pre-effect proof (`GET /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/receipt`) */ + snapshotPreparationReceipt: (sessionID: string, activityID: string, options?: RequestOptions): Promise => + caller.call("v1.session.snapshotPreparationReceipt", { path: { sessionID, activityID } }, options), /** Review cancellation of an exited preparation using its exact native lifetime proof (`POST /api/v1/sessions/{sessionID}/rewind/preparations/{activityID}/review`) */ snapshotPreparationReview: (sessionID: string, activityID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotPreparationReview", { path: { sessionID, activityID } }, options), /** List retained preparation candidates without native access; review checks eligibility (`GET /api/v1/sessions/{sessionID}/rewind/preparations`) */ - snapshotPreparations: (sessionID: string, options?: RequestOptions): Promise => - caller.call("v1.session.snapshotPreparations", { path: { sessionID } }, options), + snapshotPreparations: (sessionID: string, query?: Operations["v1.session.snapshotPreparations"]["query"], options?: RequestOptions): Promise => + caller.call("v1.session.snapshotPreparations", { path: { sessionID }, query }, options), /** Inspect retained filesystem restore journals without acknowledging or replaying native effects (`GET /api/v1/sessions/{sessionID}/rewind/recovery`) */ snapshotRecovery: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.snapshotRecovery", { path: { sessionID } }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 274944114..6a1aeb050 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -3694,6 +3694,11 @@ export type PrDescriptionRequest = { base?: string | null; }; +/** A discovery response always contains `data`, including an explicit absent receipt. */ +export type PreparationReceiptData = { + data: SnapshotPreparationReceipt | null; +}; + /** Product telemetry is not implemented and always off. */ export type ProductTelemetry = { enabled: boolean; diff --git a/sdk/typescript/test/rewind.test.ts b/sdk/typescript/test/rewind.test.ts index 8a0e2e03f..7674c17ae 100644 --- a/sdk/typescript/test/rewind.test.ts +++ b/sdk/typescript/test/rewind.test.ts @@ -112,3 +112,13 @@ test("snapshot recovery request discovery is a scoped read without acknowledgeme assert.equal(calls[0]!.url.searchParams.get("journal_id"), request.journal_id); assert.ok(!calls[0]!.body); }); + +test("preparation history and receipt discovery are opt-in reads without cancellation replay", async () => { + const { client, calls } = mockClient(() => json(200, { data: null })); + await client.request("v1.session.snapshotPreparations", { path: { sessionID: "ses_owner" }, query: { include_cancelled: true } }); + const absent: Awaited> = null; + assert.equal(await client.session.snapshotPreparationReceipt("ses_owner", "op_one"), absent); + assert.deepEqual(calls.map(call => call.method), ["GET", "GET"]); + assert.equal(calls[0]!.url.searchParams.get("include_cancelled"), "true"); + assert.equal(calls[1]!.url.pathname, "/api/v1/sessions/ses_owner/rewind/preparations/op_one/receipt"); +}); From 86d505aa887d6efe8c0fc817d3bf5a20b4d113d7 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 09:32:40 -0400 Subject: [PATCH 41/87] Test preparation recovery across terminal process death --- crates/cyber-cli/tests/branches_cli.rs | 52 +++- .../tests/support/tui_preparation_driver.py | 262 ++++++++++++++++++ .../implementation.md | 63 +++++ .../complete-daily-coding-surfaces/tasks.md | 12 +- 4 files changed, 381 insertions(+), 8 deletions(-) create mode 100644 crates/cyber-cli/tests/support/tui_preparation_driver.py diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index de25286a1..78932bccf 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -785,17 +785,24 @@ fn managed_keep_and_rollback_after_real_partial_restore_kill_preserve_owner_file #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_restart() { - cancelled_pre_effect_preparation(false); + cancelled_pre_effect_preparation(false, false); } #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn exited_origin_preparation_before_source_admission_is_cancelled_after_real_kill() { - cancelled_pre_effect_preparation(true); + cancelled_pre_effect_preparation(true, false); } +#[test] #[cfg(any(target_os = "macos", target_os = "linux"))] -fn cancelled_pre_effect_preparation(published: bool) { +fn terminal_preparation_recovery_survives_client_kill_with_a_committed_reply_withheld() { + cancelled_pre_effect_preparation(false, true); + cancelled_pre_effect_preparation(true, true); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn cancelled_pre_effect_preparation(published: bool, terminal: bool) { let model = scripted_model::ScriptedModel::start(reply); let env = Env::new(&model); let repo = env.root.join("repo"); @@ -832,7 +839,9 @@ fn cancelled_pre_effect_preparation(published: bool) { let request_id = "before-origin/ owner #?%"; let body = json!({"confirmation":review["confirmation"],"request_id":request_id}); assert_preparation_client_refusals(&env, &id, &activity, &body); - let receipt = json(&preparation_confirm_cli(&env, &id, &activity, &body)); + let receipt = preparation_with_client(&env, &id, &activity, &body, terminal); + let request_id = receipt["request_id"].as_str().unwrap(); + let body = json!({"confirmation":receipt["confirmation"],"request_id":request_id}); assert_preparation_resource_digest(&review, &receipt, published); assert_eq!( json(&preparation_request_cli( @@ -877,6 +886,41 @@ fn cancelled_pre_effect_preparation(published: bool) { env.exec(Some(&id), "after preparation cancellation"); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn preparation_with_client( + env: &Env, + session: &str, + activity: &str, + body: &Value, + terminal: bool, +) -> Value { + if terminal { + terminal_preparation(env, session, activity) + } else { + json(&preparation_confirm_cli(env, session, activity, body)) + } +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn terminal_preparation(env: &Env, session: &str, activity: &str) -> Value { + let template = env.command(&[]); + let environment = template + .get_envs() + .filter_map(|(key, value)| value.map(|value| (key, value))); + let output = Command::new("python3") + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/tui_preparation_driver.py" + )) + .args([env!("CARGO_BIN_EXE_cyber"), session, activity]) + .env_clear() + .envs(environment) + .current_dir(env.root.join("repo")) + .output() + .unwrap(); + json(&output) +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_preparation_history(env: &Env, session: &str, activity: &str, receipt: &Value) { let current = json(&env.cyber(&[ diff --git a/crates/cyber-cli/tests/support/tui_preparation_driver.py b/crates/cyber-cli/tests/support/tui_preparation_driver.py new file mode 100644 index 000000000..d17b32977 --- /dev/null +++ b/crates/cyber-cli/tests/support/tui_preparation_driver.py @@ -0,0 +1,262 @@ +"""Drive the production TUI in a PTY; withhold one committed confirmation reply. + +Only the caller's private fixture registration is redirected. No production fault +hook is used. The Rust caller supplies the orphaned native preparation and checks +source bytes, immutable receipts, messages and inference before/after this driver. +""" +import fcntl +import http.client +import http.server +import json +import os +from pathlib import Path +import pty +import re +import select +import signal +import socketserver +import struct +import subprocess +import sys +import termios +import threading +import time +from urllib.parse import urlsplit + + +class Proxy(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + + def __init__(self, upstream): + super().__init__(("127.0.0.1", 0), Forward) + self.upstream = upstream + self.calls = [] + self.held = threading.Event() + self.release = threading.Event() + self.receipt = None + + +class Forward(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass # Owner authorization headers must never reach test logs. + + def do_GET(self): + self.forward() + + def do_POST(self): + self.forward() + + def forward(self): + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + call = {"method": self.command, "path": self.path, "done": False} + self.server.calls.append(call) + upstream = http.client.HTTPConnection(self.server.upstream, timeout=45) + try: + upstream.request(self.command, self.path, body, dict(self.headers)) + response = upstream.getresponse() + call["status"] = response.status + if self.command == "POST" and "/rewind/preparations/" in self.path and self.path.endswith("/confirm"): + payload = json.loads(response.read()) + assert response.status == 200, payload + self.server.receipt = payload["data"] + assert self.server.receipt["request_id"] == json.loads(body)["request_id"] + self.server.held.set() + assert self.server.release.wait(45), "test did not release withheld reply" + return # The killed TUI never receives even response headers. + self.send_response(response.status) + for name, value in response.getheaders(): + if name.lower() not in ("connection", "transfer-encoding", "content-length"): + self.send_header(name, value) + self.send_header("Connection", "close") + self.end_headers() + self.close_connection = True + while block := response.read1(65536): + self.wfile.write(block) + self.wfile.flush() + call["done"] = True + except (BrokenPipeError, ConnectionResetError): + pass # PTY owner termination also closes its event stream. + finally: + upstream.close() + + +class Terminal: + def __init__(self, binary, session): + master, slave = pty.openpty() + fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 50, 160, 0, 0)) + environment = dict(os.environ, TERM="xterm-256color", NO_COLOR="1") + environment.pop("CI", None) + self.child = subprocess.Popen( + [binary, "--resume", session], stdin=slave, stdout=slave, stderr=slave, + env=environment, start_new_session=True, + ) + os.close(slave) + self.master = master + self.output = bytearray() + + def pump(self, seconds=0.25): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if not select.select([self.master], [], [], max(0, min(0.05, deadline - time.monotonic())))[0]: + continue + try: + block = os.read(self.master, 65536) + except OSError: + return + if not block: + return + self.output.extend(block) + # Bound diagnostics, including terminals that keep animating. + del self.output[:-1024 * 1024] + if b"\x1b[6n" in block: + self.send(b"\x1b[1;1R") + if b"\x1b[?u" in block: + self.send(b"\x1b[?0u") + if b"\x1b[c" in block: + self.send(b"\x1b[?1;2c") + + def text(self): + return re.sub(rb"\x1b(?:\[[0-?]*[ -/]*[@-~]|\].*?(?:\x07|\x1b\\))", b"", bytes(self.output), flags=re.DOTALL) + + def send(self, keys): + os.write(self.master, keys) + + def wait(self, predicate, description): + deadline = time.monotonic() + 30 + while not predicate(): + assert self.child.poll() is None, f"TUI exited during {description}: {self.text()[-3000:]!r}" + assert time.monotonic() < deadline, f"TUI timed out during {description}: {self.text()[-3000:]!r}" + self.pump() + self.pump() # Let the real runner apply the completed HTTP response. + + def stop(self, kill=False): + if self.child.poll() is None: + self.child.send_signal(signal.SIGKILL if kill else signal.SIGTERM) + self.child.wait(timeout=10) + self.pump(0.05) + os.close(self.master) + if kill: + assert self.child.returncode == -signal.SIGKILL + else: + assert self.child.returncode == 0, bytes(self.output[-5000:]) + + +def completed(proxy, suffix, success=True): + return sum( + call["done"] and call["path"].endswith(suffix) + and (not success or call["status"] == 200) + for call in proxy.calls + ) + + +def open_preparations(terminal, proxy): + history = completed(proxy, "/preparations?include_cancelled=true") + receipt = completed(proxy, "/receipt") + terminal.wait(lambda: b"ownerdraft" in re.sub(rb"\s+", b"", terminal.text()), "saved draft rendering") + terminal.send(b"\x1b") + terminal.pump(0.1) + terminal.send(b"\x1b") + # C is ignored while rewind loads; retrying this read-only entry never + # confirms source work. P must also work after journal inspection fails on + # a genuinely unconstructed pre-origin namespace. + for _ in range(6): + terminal.pump(0.5) + terminal.send(b"c") + terminal.pump() + if b"SNAPSHOTRECOVERY" in re.sub(rb"\s+", b"", terminal.text()): + break + terminal.wait(lambda: b"SNAPSHOTRECOVERY" in re.sub(rb"\s+", b"", terminal.text()), "recovery entry") + for _ in range(6): + terminal.send(b"p") + terminal.pump(0.5) + if completed(proxy, "/preparations?include_cancelled=true") > history: + break + terminal.wait(lambda: completed(proxy, "/preparations?include_cancelled=true") > history, "preparation history") + terminal.send(b"\r") + terminal.wait(lambda: completed(proxy, "/receipt") > receipt, "exact receipt discovery") + + +def positive_review(terminal, proxy): + for attempt in range(3): + terminal.wait(lambda: completed(proxy, "/review", success=False) > attempt, "pre-effect review response") + reply = [call for call in proxy.calls if call["done"] and call["path"].endswith("/review")][-1] + if reply["status"] == 200: + return attempt + 1 + assert reply["status"] == 409, reply + # A busy native namespace may refuse a read-only review. Explicit Enter + # requests fresh discovery/review; no confirmation has been submitted. + terminal.send(b"\r") + raise AssertionError("native pre-effect review stayed busy") + + +def exercise(binary, session, activity, proxy): + first = Terminal(binary, session) + try: + open_preparations(first, proxy) + review_attempts = positive_review(first, proxy) + first.send(b"\r") + first.wait(proxy.held.is_set, "committed confirmation with reply withheld") + receipt = proxy.receipt + assert receipt["activity_id"] == activity + first.stop(kill=True) + finally: + if first.child.poll() is None: + first.stop(kill=True) + proxy.release.set() + second = Terminal(binary, session) + try: + open_preparations(second, proxy) + second.send(b"\r") # Outcome Enter must never replay cancellation. + second.pump() + key = receipt["request_id"] + second.send(b"r") + second.wait(lambda: completed(proxy, f"/requests/{key}") == 1, "original request lookup") + assert not any(call["path"].endswith("/acknowledge") for call in proxy.calls), "receipt discovery acknowledged native metadata without an explicit action" + second.send(b"a") + second.wait(lambda: completed(proxy, f"/requests/{key}/acknowledge") == 1, "explicit reconciliation") + second.send(b"\x1b") + second.pump() + second.stop() + finally: + if second.child.poll() is None: + second.stop(kill=True) + confirmations = [call for call in proxy.calls if call["path"].endswith("/confirm")] + assert len(confirmations) == 1, "fresh terminal replayed cancellation" + assert completed(proxy, f"/requests/{key}/acknowledge") == 1 + assert completed(proxy, "/review", success=False) == review_attempts, "fresh terminal requested a new cancellation review" + return receipt + + +def main(): + binary, session, activity = sys.argv[1:] + state = Path(os.environ["CYBER_HOME"]) / "state" + path = state / "server.json" + original = path.read_bytes() + registration = json.loads(original) + draft = "owner draft [Pasted text #1 +1 lines]" + chips = [{"kind": "text", "number": 1, "payload": {"source": "text", "text": "owner attachment"}}] + kv_path = state / "kv.json" + kv = json.loads(kv_path.read_text()) if kv_path.exists() else {} + kv.update({f"draft:{session}": draft, f"chips:{session}": chips}) + kv_path.write_text(json.dumps(kv)) + proxy = Proxy(urlsplit(registration["url"]).netloc) + threading.Thread(target=proxy.serve_forever, daemon=True).start() + registration["url"] = f"http://127.0.0.1:{proxy.server_port}" + path.write_text(json.dumps(registration)) + try: + receipt = exercise(binary, session, activity, proxy) + saved = json.loads(kv_path.read_text()) + assert saved[f"draft:{session}"] == draft, "TUI recovery changed the draft" + assert saved[f"chips:{session}"] == chips, "TUI recovery changed attachment payloads" + print(json.dumps(receipt)) + finally: + path.write_bytes(original) + proxy.release.set() + proxy.shutdown() + proxy.server_close() + + +if __name__ == "__main__": + main() diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 4ce77c184..fbf6a5b2c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3078,3 +3078,66 @@ attribution. Full workspace, native preparation TUI process-kill, additional Linux/macOS architectures, Windows and source-bound CI were not run here; this is local evidence, not complete M7/M8 or accepted delivery. No PR/push is made by this increment and no parent task or issue is closed. + +## Production TUI process death with a committed preparation reply withheld (2026-11-07) + +A new default Linux/macOS branch integration case drives the actual `cyber --resume` +binary through a POSIX pseudo-terminal, using production terminal setup, keyboard +dispatch, event loop and HTTP transport. Its Python standard-library driver lives +under CLI test support; the Rust fixture retains native source and Store assertions. +It uses the same real service-kill boundaries before origin and after origin but +before source admission, with exact retained lifetime evidence. No production fault +hook or recovery admission behavior changes. + +A loopback proxy redirects only the fixture's private registration URL, forwards +the original owner authentication, and withholds all confirmation response headers +and bytes after receiving a successful committed receipt from the real server. +The test observes SIGKILL termination of the first TUI, launches a new production +TUI, and enters preparation recovery through Esc-Esc/C/P and actual Enter keys. +The new process discovers the existing receipt, makes outcome Enter inert, looks +up the original key with R and explicitly reconciles it with A. The proxy requires +exactly one cancellation POST across both terminal processes and no additional +review after committed receipt discovery. Positive lookup and acknowledgement +responses must be HTTP 200, and no acknowledgement request may precede explicit A. +Transient busy read-only reviews may be explicitly rediscovered/reviewed up to +three times before confirmation; no mutation is retried. Read-only C/P entry also +handles loading and journal-inspection refusal from an unconstructed pre-origin +namespace; that absence never supplies cancellation proof. The proxy restores registration and releases held responses in cleanup. + +The saved draft and text-attachment payload survive the killed terminal and the +second terminal's graceful exit. The shared native fixture also verifies immutable +receipt/resource identity, restart lookup and idempotent confirmation, later owner +symlink/target bytes, other source bytes, unchanged messages and no inference. +The final focused two-boundary case passed locally on macOS arm64 in 74.23 seconds. + +Code-review skill inspection covered private registration/credential handling, +PTY ownership and bounded diagnostics/timeouts, source assertions, exact request +correlation, read-only retry versus cancellation replay, child cleanup and honest +platform scope. Rust Clippy's new fixture 16/15 finding was resolved by separating +client confirmation dispatch. The cognitive-complexity skill ran complexipy 8.3.0 +from an isolated temporary environment: the Python driver's maximum was 13 in its +PTY pump, with no analyzer warnings. No unresolved critical/major review finding +was identified. Other native platforms, writer/native-acknowledgement/reclaim crash +cuts, managed Git source admission, partial constructors and safe reclamation +remain incomplete; this local acceptance does not close M7/M8 or either issue. + +Verification: the serial branch/recovery invocation passed all 17 existing cases +and failed the new PTY case because its driver incorrectly required successful +journal inspection before P. After correcting only that synchronization assumption, +the final focused case passed both native boundaries with the stricter positive +HTTP/explicit-acknowledgement checks. The full 18-case invocation was not repeated +after that driver fix; the existing cases do not use this Python driver. All-target +CLI Clippy, Python compilation, complexity analysis, format/diff checks and pinned +strict OpenSpec (117 items) passed. Spec lint reported 0 errors/25 warnings, without +unchanged-main attribution. C deprecation and oversized-unwind linker warnings +were observed without origin attribution. All Cargo tests/Clippy used +CARGO_INCREMENTAL=0 and --locked. No separate SDK/server/TUI-library/full-workspace +or additional native platform/CI checks ran for this test-only increment. It makes +no PR/push and closes no parent task or issue. + +Reproduce the focused real-terminal case with Python 3 and its POSIX PTY standard +library available (it runs by default in Linux/macOS branch integration tests): + +```sh +CARGO_INCREMENTAL=0 cargo test --locked -p cyber-cli --test branches_cli terminal_preparation_recovery -- --test-threads=1 +``` diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 053c38cd2..a80ebd6c7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -513,7 +513,11 @@ attachments and conversation. Opt-in history keeps default pending listing intac exact receipt discovery delegates to the validated original committed pair. Controller and actual HTTP transport coverage accompanies native CLI service-kill cuts before/after origin publication, including restarted history/discovery and -foreign-Session refusal. Actual native preparation cancellation driven by a killed -TUI process remains a separate acceptance gap. M7/M8 and their parent tasks stay -open alongside partial constructors, managed Git admission, safe reclamation and -complete compatibility/platform evidence. +foreign-Session refusal. The subsequent real-binary PTY test now covers native +pre-origin and published-origin cancellation with the committed confirmation +response withheld, SIGKILL of the TUI, and fresh-process discovery/lookup/explicit +acknowledgement without another cancellation. It preserves saved draft attachment +payloads and later source symlinks/bytes, messages and inference counts. These are +local macOS arm64 positives; other platform/cut combinations remain unproven. +M7/M8 and their parent tasks stay open alongside partial constructors, managed Git +admission, safe reclamation and complete compatibility/platform evidence. From 793aa089fa94271d13c02e0d504f34d1f84669ca Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 11:02:51 -0400 Subject: [PATCH 42/87] Reclaim acknowledged file-copy restore payloads --- crates/cyber-app/src/lib.rs | 16 +- crates/cyber-app/src/snapshot_reclamation.rs | 128 +++ .../cyber-app/tests/snapshot_reclamation.rs | 861 ++++++++++++++++++ .../tests/support/tui_preparation_driver.py | 23 +- crates/cyber-server/src/runtime/events.rs | 2 + crates/cyber-server/src/runtime/mod.rs | 1 + .../src/runtime/snapshot_reclamation.rs | 283 ++++++ crates/cyber-snapshot/src/copy/lifetime.rs | 20 + crates/cyber-snapshot/src/copy/mod.rs | 1 + crates/cyber-snapshot/src/copy/ownership.rs | 5 + crates/cyber-snapshot/src/copy/reclamation.rs | 419 +++++++++ crates/cyber-snapshot/src/lib.rs | 153 +++- crates/cyber-snapshot/src/reclamation.rs | 25 + ...0261107030000_snapshot_reference_index.sql | 28 + crates/cyber-store/src/migrate.rs | 96 +- docs/rewind.md | 16 +- .../implementation.md | 124 +++ .../managed-recovery.md | 41 + .../specs/storage-events/spec.md | 15 + .../complete-daily-coding-surfaces/tasks.md | 11 + 20 files changed, 2239 insertions(+), 29 deletions(-) create mode 100644 crates/cyber-app/src/snapshot_reclamation.rs create mode 100644 crates/cyber-app/tests/snapshot_reclamation.rs create mode 100644 crates/cyber-server/src/runtime/snapshot_reclamation.rs create mode 100644 crates/cyber-snapshot/src/copy/reclamation.rs create mode 100644 crates/cyber-snapshot/src/reclamation.rs create mode 100644 crates/cyber-store/migrations/20261107030000_snapshot_reference_index.sql diff --git a/crates/cyber-app/src/lib.rs b/crates/cyber-app/src/lib.rs index 4a6acacda..f1d20bfe1 100644 --- a/crates/cyber-app/src/lib.rs +++ b/crates/cyber-app/src/lib.rs @@ -16,6 +16,7 @@ mod registration; pub mod retention; mod server; mod services; +pub mod snapshot_reclamation; pub mod update_check; use std::path::PathBuf; @@ -325,17 +326,11 @@ impl App { let loader = Arc::clone(&config); let snapshot_store = store.clone(); let references: Arc = Arc::new(move || { - let rows = snapshot_store.read(|db| { - let mut query = db.prepare("SELECT DISTINCT j.value FROM event e,json_tree(e.data) j WHERE j.type='text' AND j.value LIKE 'copy:%' LIMIT 100001")?; - let rows = query.query_map([], |row|row.get::<_,String>(0))?.collect::,_>>()?; - Ok(rows) - }).map_err(|error|error.to_string())?; - if rows.len() > 100000 { - return Err("Snapshot reference limit exceeded; pruning refused".into()); - } - Ok(rows.into_iter().collect()) + snapshot_store + .read(cyber_server::runtime::snapshot_reclamation::references) + .map_err(|error| error.to_string()) }); - let snaps = cyber_snapshot::GitSnapshots::with_policy( + let snaps = cyber_snapshot::GitSnapshots::with_reclamation( opts.paths.data.clone(), Arc::new(move |p| loader(p).map(|(v, _)| v).unwrap_or(Value::Null)), vec![ @@ -345,6 +340,7 @@ impl App { opts.paths.tmp.clone(), ], Some(references), + Some(snapshot_reclamation::policy(store.clone())), ); snaps.spawn_gc(); snaps diff --git a/crates/cyber-app/src/snapshot_reclamation.rs b/crates/cyber-app/src/snapshot_reclamation.rs new file mode 100644 index 000000000..588b459d1 --- /dev/null +++ b/crates/cyber-app/src/snapshot_reclamation.rs @@ -0,0 +1,128 @@ +//! Compose native cleanup with the same serialized writer that owns references. +use cyber_server::runtime::{ + SnapshotOrigin, + snapshot_reclamation::{self as records, Completion, Plan, Record}, +}; +use cyber_snapshot::reclamation::{Collect, Delete, Policy, Settlement}; +use cyber_store::{Expected, NewEvent, Store, StoreError}; +use std::sync::Arc; + +pub(crate) struct Reclamation(pub Arc); +/// Bind native maintenance to this owned Store. Embedders must use the same +/// Store as their runtime; this grants no HTTP or source-file capability. +pub fn policy(store: Arc) -> Arc { + Arc::new(Reclamation(store)) +} +fn invalid(error: impl std::fmt::Display) -> StoreError { + StoreError::InvalidAppend(error.to_string()) +} + +impl Policy for Reclamation { + fn receipt(&self, origin: &SnapshotOrigin) -> Result, String> { + let origin = origin.clone(); + self.0 + .read(move |db| { + let tx = db.unchecked_transaction()?; + let Some(receipt) = records::restore_receipt(&tx, &origin)? else { + return Ok(None); + }; + let settled_at_ms = tx.query_row( + "SELECT time FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&origin.activity_id], + |row| row.get(0), + )?; + tx.commit()?; + Ok(Some(Settlement { + receipt, + settled_at_ms, + })) + }) + .map_err(|error| error.to_string()) + } + fn record( + &self, + receipt: &cyber_server::runtime::SnapshotRestoreReceipt, + ) -> Result, String> { + let receipt = receipt.clone(); + self.0 + .read(move |db| { + let tx = db.unchecked_transaction()?; + let record = records::lookup(&tx, &receipt)?; + tx.commit()?; + Ok(record) + }) + .map_err(|error| error.to_string()) + } + fn reclaim(&self, plan: Plan, delete: Delete) -> Result { + plan.validate().map_err(|error| error.to_string())?; + let id = Plan::id(&plan.receipt).map_err(|error| error.to_string())?; + let existing = self.record(&plan.receipt)?; + if let Some(existing) = existing { + if existing.plan != plan { + return Err("Reclamation plan differs from retained request".into()); + } + if existing.completed { + return Ok(0); + } + } else { + let prepare = plan.clone(); + self.0 + .append_checked(&id, Expected::Seq(-1), move |db| { + prepare.authorize(db)?; + let event = NewEvent::new( + records::PLANNED, + serde_json::to_value(&prepare).map_err(invalid)?, + ); + Ok((vec![event], ())) + }) + .map_err(|error| error.to_string())?; + } + let consume = plan.clone(); + let removed = self + .0 + .transaction(move |db| { + let record = records::lookup(db, &consume.receipt)? + .ok_or_else(|| invalid("Missing committed reclamation plan"))?; + if record.completed || record.plan != consume { + return Err(invalid("Reclamation request changed before deletion")); + } + consume.authorize(db)?; + // No network, child processes, callbacks into Store or source-file + // work. Native locks/pins remain owned by this bounded closure. + delete().map_err(invalid) + }) + .map_err(|error| error.to_string())?; + let completion = Completion { + plan_id: id.clone(), + removed_bytes: removed, + }; + self.0 + .append_checked(&id, Expected::Seq(0), move |db| { + plan.authorize(db)?; + if completion.removed_bytes != plan.total_bytes()? { + return Err(invalid("Reclamation byte accounting changed")); + } + Ok(( + vec![NewEvent::new( + records::COMPLETED, + serde_json::to_value(completion).map_err(invalid)?, + )], + (), + )) + }) + .map_err(|error| error.to_string())?; + Ok(removed) + } + fn collect(&self, collect: Collect) -> Result<(), String> { + self.0 + .transaction(move |db| collect(&records::references(db)?).map_err(invalid)) + .map_err(|error| error.to_string()) + } + fn refused(&self, directory: &str, error: &str) { + cyber_core::log::warn( + "snapshot", + "Snapshot reclamation refused; retained evidence is protected", + serde_json::json!({"directory":directory,"reason":error}), + ); + } +} diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs new file mode 100644 index 000000000..9f7949d0f --- /dev/null +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -0,0 +1,861 @@ +#![cfg(any(target_os = "macos", target_os = "linux"))] +//! Real App restores/receipts and native private payloads; no source writes by cleanup. +#[path = "support/scripted_model.rs"] +mod scripted_model; +use cyber_app::{App, AppOptions}; +use cyber_client::Client; +use cyber_core::paths::{DatabaseLocation, Paths}; +use cyber_server::runtime::{SnapshotRestoreReceipt, Snapshots, snapshot_reclamation as records}; +use cyber_snapshot::{ + GitSnapshots, + reclamation::{Collect, Delete, Policy, Settlement}, +}; +use serde_json::{Value, json}; +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, + time::Duration, +}; + +struct Fixture { + _temp: tempfile::TempDir, + app: App, + client: Client, + model: scripted_model::ScriptedModel, + paths: Paths, + project: PathBuf, + session: String, + receipt: SnapshotRestoreReceipt, + journal: PathBuf, + config: Arc>, + snapshots: Arc, +} +impl Fixture { + async fn new() -> Self { + Self::in_directory(tempfile::tempdir().unwrap()).await + } + async fn in_directory(temp: tempfile::TempDir) -> Self { + let root = temp.path().canonicalize().unwrap(); + let paths = Paths { + data: root.join("data"), + config: root.join("config"), + state: root.join("state"), + cache: root.join("cache"), + tmp: root.join("tmp"), + }; + paths.ensure().unwrap(); + let project = root.join("project"); + std::fs::create_dir(&project).unwrap(); + let model = scripted_model::ScriptedModel::start(|_| json!({"content":"Done."})); + std::fs::write(paths.config.join("cyber.jsonc"), model.config().to_string()).unwrap(); + let app = App::build(AppOptions { + paths: paths.clone(), + home: root.join("home"), + database: DatabaseLocation::File(paths.data.join("cyber.db")), + default_directory: project.clone(), + sandbox_policy: None, + snapshots: true, + interactive: true, + password: None, + mcp_credentials: None, + remote_skill_fetcher: None, + workflow_cores: None, + isolation: None, + }) + .await + .unwrap(); + let client = Client::embedded(app.embedded()).at(project.to_str().unwrap()); + let session = client + .post( + "/sessions", + json!({"model":"test/main","mode":"accept-edits"}), + ) + .await + .unwrap()["data"]["id"] + .as_str() + .unwrap() + .to_owned(); + for index in 0..2 { + std::fs::write( + project.join(format!("file_{index}")), + "before\n".repeat(4096), + ) + .unwrap(); + } + prompt(&app, &client, &session, "before snapshot", 1).await; + let messages = client + .get(&format!("/sessions/{session}/messages?order=asc")) + .await + .unwrap(); + let first = messages["data"] + .as_array() + .unwrap() + .iter() + .find(|message| message["kind"] == "user") + .unwrap()["id"] + .as_str() + .unwrap() + .to_owned(); + for index in 0..2 { + std::fs::write( + project.join(format!("file_{index}")), + "after!\n".repeat(4096), + ) + .unwrap(); + } + prompt(&app, &client, &session, "after snapshot", 2).await; + let mut selection = json!({"message_id":first,"target":"code","action":"commit","expected_directory":project}); + let preview = client + .post( + &format!("/sessions/{session}/rewind/preview"), + selection.clone(), + ) + .await + .unwrap(); + selection["confirmation"] = preview["data"]["confirmation"].clone(); + client + .post(&format!("/sessions/{session}/rewind/confirm"), selection) + .await + .unwrap(); + let owner = session.clone(); + let receipt: SnapshotRestoreReceipt = app.store.read(move |db| { + let raw: String = db.query_row("SELECT json_extract(data,'$.snapshot_restore_receipt') FROM event WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_restore_receipt') IS NOT NULL ORDER BY rowid DESC LIMIT 1", [owner], |row| row.get(0))?; + Ok(serde_json::from_str(&raw).unwrap()) + }).unwrap(); + let journal = std::fs::read_dir(paths.data.join("snapshot/global")) + .unwrap() + .map(|entry| { + entry + .unwrap() + .path() + .join("journals") + .join(&receipt.origin.binding.journal_id) + }) + .find(|path| path.exists()) + .unwrap(); + assert!(journal.join("stage_0").exists() && journal.join("stage_1").exists()); + let config = Arc::new(Mutex::new(json!({"snapshots":{"retention_days":7}}))); + let snapshots = maintenance( + &paths, + config.clone(), + cyber_app::snapshot_reclamation::policy(app.store.clone()), + ); + Self { + _temp: temp, + app, + client, + model, + paths, + project, + session, + receipt, + journal, + config, + snapshots, + } + } + fn age(&self) { + let owner = self.receipt.origin.activity_id.clone(); + // Only the retention clock is aged. Receipt/native ownership bytes and + // original event sequence remain those of the actual committed restore. + self.app + .store + .transaction(move |db| { + db.execute( + "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", + rusqlite::params![8i64 * 86400000, owner], + )?; + Ok(()) + }) + .unwrap(); + } + async fn reclaim(&self) -> Result { + self.snapshots.reclaim(self.project.to_str().unwrap()).await + } + fn record(&self) -> Option { + let receipt = self.receipt.clone(); + self.app + .store + .read(move |db| records::lookup(db, &receipt)) + .unwrap() + } +} +fn maintenance( + paths: &Paths, + config: Arc>, + policy: Arc, +) -> Arc { + GitSnapshots::with_reclamation( + paths.data.clone(), + Arc::new(move |_| config.lock().unwrap().clone()), + vec![ + paths.config.clone(), + paths.state.clone(), + paths.cache.clone(), + paths.tmp.clone(), + ], + None, + Some(policy), + ) +} +async fn prompt(app: &App, client: &Client, session: &str, text: &str, expected: usize) { + client + .post( + &format!("/sessions/{session}/prompt"), + json!({"parts":[{"type":"text","text":text}]}), + ) + .await + .unwrap(); + for _ in 0..1000 { + let messages = client + .get(&format!("/sessions/{session}/messages")) + .await + .unwrap(); + let answered = messages["data"] + .as_array() + .unwrap() + .iter() + .filter(|message| message["kind"] == "assistant") + .count(); + if answered >= expected && !app.runtime.is_running(session) { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("scripted prompt did not finish"); +} + +#[tokio::test] +async fn acknowledged_ordinary_payloads_are_reclaimed_without_source_or_reference_image_changes() { + let f = Fixture::new().await; + f.age(); + let messages = f + .client + .get(&format!("/sessions/{}/messages", f.session)) + .await + .unwrap(); + let inference = f.model.requests().len(); + let intent = std::fs::read(f.journal.join("intent.json")).unwrap(); + let files = f.journal.parent().unwrap().parent().unwrap().join("files"); + let images = image_bytes(&files); + let target = f.project.parent().unwrap().join("later-owner"); + std::fs::write(&target, "later owner bytes").unwrap(); + std::fs::remove_file(f.project.join("file_0")).unwrap(); + std::os::unix::fs::symlink(&target, f.project.join("file_0")).unwrap(); + assert_eq!( + f.reclaim().await.unwrap(), + 2 * ("after!\n".len() * 4096) as u64 + ); + assert!(!f.journal.join("stage_0").exists() && !f.journal.join("stage_1").exists()); + assert_eq!( + std::fs::read(f.journal.join("intent.json")).unwrap(), + intent + ); + assert_eq!(image_bytes(&files), images); + assert!(f.record().unwrap().completed); + assert_eq!(f.reclaim().await.unwrap(), 0); + f.client + .post( + &format!( + "/sessions/{}/rewind/restores/{}/acknowledge", + f.session, f.receipt.origin.activity_id + ), + json!({}), + ) + .await + .unwrap(); + assert!( + std::fs::symlink_metadata(f.project.join("file_0")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_to_string(target).unwrap(), + "later owner bytes" + ); + assert_eq!( + f.client + .get(&format!("/sessions/{}/messages", f.session)) + .await + .unwrap(), + messages + ); + assert_eq!(f.model.requests().len(), inference); + f.app.runtime.shutdown().await; +} +fn image_bytes(path: &std::path::Path) -> std::collections::BTreeMap> { + std::fs::read_dir(path) + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + ( + entry.file_name().to_str().unwrap().to_owned(), + std::fs::read(entry.path()).unwrap(), + ) + }) + .collect() +} + +#[tokio::test] +async fn periodic_collection_applies_the_same_retention_and_receipt_admission() { + let f = Fixture::new().await; + let intent = std::fs::read(f.journal.join("intent.json")).unwrap(); + let files = f.journal.parent().unwrap().parent().unwrap().join("files"); + let images = image_bytes(&files); + f.snapshots.gc().await; + assert!(f.record().is_none() && f.journal.join("stage_0").exists()); + f.age(); + f.snapshots.gc().await; + assert!(f.record().unwrap().completed); + assert!(!f.journal.join("stage_0").exists() && !f.journal.join("stage_1").exists()); + assert_eq!( + std::fs::read(f.journal.join("intent.json")).unwrap(), + intent + ); + assert_eq!(image_bytes(&files), images); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn retention_disabled_capture_disabled_and_live_lifetime_do_not_license_cleanup() { + let f = Fixture::new().await; + assert_eq!(f.reclaim().await.unwrap(), 0); + f.age(); + *f.config.lock().unwrap() = json!({"snapshots":{"retention_days":0}}); + assert_eq!(f.reclaim().await.unwrap(), 0); + *f.config.lock().unwrap() = json!({"snapshots":false}); + assert_eq!(f.reclaim().await.unwrap(), 0); + *f.config.lock().unwrap() = json!({}); + let reader = cyber_server::workspace::mutation_lock(&f.project) + .unwrap() + .lock_owned() + .await; + assert!(f.reclaim().await.is_err()); + assert!(f.record().is_none()); + drop(reader); + let lifetime = f.receipt.checkout.as_ref().unwrap().lifetime.clone(); + let worker = f + .journal + .parent() + .unwrap() + .parent() + .unwrap() + .join("workers") + .join(lifetime.owner.activity_id); + let held = std::fs::File::open(worker).unwrap(); + held.try_lock().unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(f.record().is_none() && f.journal.join("stage_0").exists()); + drop(held); + assert!(f.reclaim().await.unwrap() > 0); + f.app.runtime.shutdown().await; +} + +struct Cut { + inner: Arc, + after_delete: bool, + pause: Option, +} +impl Policy for Cut { + fn receipt( + &self, + origin: &cyber_server::runtime::SnapshotOrigin, + ) -> Result, String> { + self.inner.receipt(origin) + } + fn record(&self, receipt: &SnapshotRestoreReceipt) -> Result, String> { + self.inner.record(receipt) + } + fn reclaim(&self, plan: records::Plan, delete: Delete) -> Result { + let after = self.after_delete; + let pause = self.pause.clone(); + self.inner.reclaim( + plan, + Box::new(move || { + if after { + delete()?; + } + if let Some(marker) = pause { + std::fs::write(marker, b"native cleanup boundary reached").unwrap(); + loop { + std::thread::park(); + } + } + Err("injected cleanup interruption".into()) + }), + ) + } + fn collect(&self, collect: Collect) -> Result<(), String> { + self.inner.collect(collect) + } + fn refused(&self, directory: &str, error: &str) { + self.inner.refused(directory, error); + } +} +#[tokio::test] +async fn durable_plan_survives_interruption_before_and_after_private_deletion() { + for after_delete in [false, true] { + let f = Fixture::new().await; + f.age(); + let policy = Arc::new(Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after_delete, + pause: None, + }); + let cut = maintenance(&f.paths, f.config.clone(), policy); + assert!(cut.reclaim(f.project.to_str().unwrap()).await.is_err()); + let record = f.record().unwrap(); + assert!(!record.completed && !record.plan.files.is_empty()); + assert_eq!(f.journal.join("stage_0").exists(), !after_delete); + assert!(f.reclaim().await.unwrap() > 0); + assert!(f.record().unwrap().completed); + assert!(!f.journal.join("stage_0").exists()); + f.app.runtime.shutdown().await; + } +} + +// Invoked in a separate test process only; ordinary suite discovery is inert. +#[tokio::test] +async fn reclamation_process_worker() { + let Ok(control) = std::env::var("CYBER_RECLAMATION_PROCESS_FIXTURE") else { + return; + }; + let control = PathBuf::from(control); + let f = Fixture::in_directory(tempfile::tempdir_in(&control).unwrap()).await; + f.age(); + std::fs::write( + control.join("fixture.json"), + serde_json::to_vec(&json!({ + "root": f._temp.path(), "project": f.project, + "journal": f.journal, "receipt": f.receipt, + })) + .unwrap(), + ) + .unwrap(); + let policy = Arc::new(Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after_delete: std::env::var("CYBER_RECLAMATION_AFTER_DELETE").unwrap() == "true", + pause: Some(control.join("ready")), + }); + let cut = maintenance(&f.paths, f.config.clone(), policy); + let result = cut.reclaim(f.project.to_str().unwrap()).await; + panic!("worker returned without reaching kill boundary: {result:?}"); +} + +#[tokio::test] +async fn actual_process_death_preserves_plan_and_resumes_without_source_replay() { + use std::os::unix::process::ExitStatusExt; + for after_delete in [false, true] { + let control = tempfile::tempdir().unwrap(); + let log = std::fs::File::create(control.path().join("worker.log")).unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "reclamation_process_worker", "--nocapture"]) + .env("CYBER_RECLAMATION_PROCESS_FIXTURE", control.path()) + .env("CYBER_RECLAMATION_AFTER_DELETE", after_delete.to_string()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(60); + let ready = loop { + if control.path().join("ready").exists() { + break true; + } + if child.try_wait().unwrap().is_some() || std::time::Instant::now() >= deadline { + break false; + } + tokio::time::sleep(Duration::from_millis(50)).await; + }; + // Always reap the child, including failed fixture startup/timeouts. + let _ = child.kill(); + let status = child.wait().unwrap(); + assert!( + ready, + "worker failed: {}", + std::fs::read_to_string(control.path().join("worker.log")).unwrap() + ); + assert_eq!(status.signal(), Some(9)); + let fixture: Value = + serde_json::from_slice(&std::fs::read(control.path().join("fixture.json")).unwrap()) + .unwrap(); + let root = PathBuf::from(fixture["root"].as_str().unwrap()); + let project = PathBuf::from(fixture["project"].as_str().unwrap()); + let journal = PathBuf::from(fixture["journal"].as_str().unwrap()); + let receipt: SnapshotRestoreReceipt = + serde_json::from_value(fixture["receipt"].clone()).unwrap(); + let intent = std::fs::read(journal.join("intent.json")).unwrap(); + assert_eq!(journal.join("stage_0").exists(), !after_delete); + // Install a later owner's route after death: resumption must never replay + // restore effects or overwrite it, even when all payloads are gone. + let target = root.join("later-owner"); + std::fs::write(&target, "later owner bytes").unwrap(); + std::fs::remove_file(project.join("file_0")).unwrap(); + std::os::unix::fs::symlink(&target, project.join("file_0")).unwrap(); + let paths = Paths { + data: root.join("data"), + config: root.join("config"), + state: root.join("state"), + cache: root.join("cache"), + tmp: root.join("tmp"), + }; + let store = Arc::new( + cyber_store::Store::open(cyber_store::StoreOptions::new( + DatabaseLocation::File(paths.data.join("cyber.db")), + cyber_server::runtime::Runtime::registry(), + )) + .unwrap(), + ); + let policy = cyber_app::snapshot_reclamation::policy(store.clone()); + assert!(!policy.record(&receipt).unwrap().unwrap().completed); + let snapshots = maintenance( + &paths, + Arc::new(Mutex::new(json!({"snapshots":{"retention_days":7}}))), + policy.clone(), + ); + assert!(snapshots.reclaim(project.to_str().unwrap()).await.unwrap() > 0); + assert!(policy.record(&receipt).unwrap().unwrap().completed); + assert_eq!( + snapshots.reclaim(project.to_str().unwrap()).await.unwrap(), + 0 + ); + assert!(!journal.join("stage_0").exists() && !journal.join("stage_1").exists()); + assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); + assert_eq!(std::fs::read_link(project.join("file_0")).unwrap(), target); + assert_eq!( + std::fs::read_to_string(target).unwrap(), + "later owner bytes" + ); + } +} + +#[tokio::test] +async fn cap_pressure_reclaims_only_eligible_payloads_then_retries_capture() { + let f = Fixture::new().await; + f.age(); + let namespace = f.journal.parent().unwrap().parent().unwrap(); + let cap = private_bytes(namespace) + 4096; + *f.config.lock().unwrap() = json!({"snapshots":{"retention_days":7,"fallback_max_bytes":cap}}); + let requests = f.model.requests().len(); + std::fs::write(f.project.join("zz_new"), vec![83u8; 8192]).unwrap(); + let captured = f + .snapshots + .track(f.project.to_str().unwrap()) + .await + .unwrap() + .unwrap(); + assert!(captured.tree.starts_with("copy:")); + assert!(f.record().unwrap().completed); + assert!(!f.journal.join("stage_0").exists()); + assert!(private_bytes(namespace) <= cap); + assert_eq!( + std::fs::read(f.project.join("zz_new")).unwrap(), + vec![83u8; 8192] + ); + assert_eq!(f.model.requests().len(), requests); + f.app.runtime.shutdown().await; +} +fn private_bytes(path: &std::path::Path) -> u64 { + std::fs::read_dir(path) + .unwrap() + .map(|entry| { + let path = entry.unwrap().path(); + if path.is_dir() { + private_bytes(&path) + } else { + std::fs::symlink_metadata(path).unwrap().len() + } + }) + .sum() +} + +#[tokio::test] +async fn missing_replaced_aliased_and_unknown_payloads_refuse_before_any_unlink() { + for kind in [ + "missing", + "replacement", + "alias", + "unknown", + "symlink", + "phase", + ] { + let f = Fixture::new().await; + f.age(); + corrupt_payload(&f, kind); + assert!(f.reclaim().await.is_err(), "{kind}"); + assert!(f.record().is_none(), "{kind}"); + assert!(f.journal.join("stage_1").exists(), "{kind}"); + assert_eq!( + std::fs::read_to_string(f.project.join("file_1")).unwrap(), + "before\n".repeat(4096) + ); + f.app.runtime.shutdown().await; + } +} +fn corrupt_payload(f: &Fixture, kind: &str) { + let stage = f.journal.join("stage_0"); + let owner = f.paths.data.join("owner-kept"); + match kind { + "phase" => { + let binding = &f.receipt.checkout.as_ref().unwrap().lifetime; + let path = f + .journal + .parent() + .unwrap() + .parent() + .unwrap() + .join("workers") + .join(binding.source_phase.as_ref().unwrap()); + let mut phase: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + phase["admitted"] = false.into(); + phase["origin"] = Value::Null; + std::fs::write(path, phase.to_string()).unwrap(); + } + "missing" => std::fs::remove_file(stage).unwrap(), + "replacement" => { + let bytes = std::fs::read(&stage).unwrap(); + std::fs::rename(&stage, owner).unwrap(); + std::fs::write(stage, bytes).unwrap(); + } + "alias" => std::fs::hard_link(stage, owner).unwrap(), + "unknown" => std::fs::write(f.journal.join("owner-kept"), "owner bytes").unwrap(), + "symlink" => { + std::fs::write(&owner, "owner target bytes").unwrap(); + std::fs::remove_file(&stage).unwrap(); + std::os::unix::fs::symlink(owner, stage).unwrap(); + } + _ => panic!("unknown corruption"), + } +} + +#[tokio::test] +async fn independent_unknown_owner_and_missing_location_view_keep_their_fences() { + for mode in ["same", "missing", "other", "missing_status"] { + let f = Fixture::new().await; + f.age(); + let other = f.project.parent().unwrap().join("other-location"); + std::fs::create_dir(&other).unwrap(); + let foreign = f + .client + .at(other.to_str().unwrap()) + .post("/sessions", json!({"model":"test/main"})) + .await + .unwrap()["data"]["id"] + .as_str() + .unwrap() + .to_owned(); + let source = match mode { + "same" => &f.session, + "missing" => "ses_missing", + _ => &foreign, + }; + // Retained legacy uncertainty, not a fabricated successful process. + let mut evidence = json!({"session_id":source,"status":"unknown","phase":"legacy_process"}); + if mode == "missing_status" { + evidence.as_object_mut().unwrap().remove("status"); + } + let retained = evidence.clone(); + f.app + .store + .append( + "op_legacy_unknown", + cyber_store::Expected::Seq(-1), + vec![cyber_store::NewEvent::new( + "native.activity.changed.1", + evidence, + )], + ) + .unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(f.record().is_none() && f.journal.join("stage_0").exists()); + let raw:String = f.app.store.read(|db| Ok(db.query_row("SELECT data FROM event WHERE aggregate_id='op_legacy_unknown' ORDER BY seq DESC LIMIT 1",[],|row|row.get(0))?)).unwrap(); + assert_eq!(serde_json::from_str::(&raw).unwrap(), retained); + f.app.runtime.shutdown().await; + } +} + +#[tokio::test] +async fn reference_writers_cannot_interleave_with_native_collection() { + let f = Fixture::new().await; + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + let (started, entered) = std::sync::mpsc::channel(); + let (release, gate) = std::sync::mpsc::channel(); + let collecting = std::thread::spawn(move || { + policy.collect(Box::new(move |roots| { + assert!(!roots.is_empty()); + started.send(()).unwrap(); + gate.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(()) + })) + }); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + let wrote = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (store, flag) = (f.app.store.clone(), wrote.clone()); + let writer = std::thread::spawn(move || { + store.transaction(move |_| { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }) + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!wrote.load(std::sync::atomic::Ordering::SeqCst)); + release.send(()).unwrap(); + collecting.join().unwrap().unwrap(); + writer.join().unwrap().unwrap(); + assert!(wrote.load(std::sync::atomic::Ordering::SeqCst)); + f.app.runtime.shutdown().await; +} + +async fn planned_without_deletion(f: &Fixture) { + let policy = Arc::new(Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after_delete: false, + pause: None, + }); + assert!( + maintenance(&f.paths, f.config.clone(), policy) + .reclaim(f.project.to_str().unwrap()) + .await + .is_err() + ); + assert!(!f.record().unwrap().completed); +} +#[tokio::test] +async fn frozen_plan_accepts_partial_progress_but_not_replaced_remaining_images() { + for replace in [false, true] { + let f = Fixture::new().await; + f.age(); + planned_without_deletion(&f).await; + // Storage-state cut of one already-planned private unlink, not a claim + // of a process-kill acceptance cut. + std::fs::remove_file(f.journal.join("stage_0")).unwrap(); + if replace { + let path = f.journal.join("stage_1"); + let bytes = std::fs::read(&path).unwrap(); + std::fs::rename(&path, f.paths.data.join("preserved-original")).unwrap(); + std::fs::write(&path, bytes).unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(!f.record().unwrap().completed && path.exists()); + } else { + assert!(f.reclaim().await.unwrap() > 0); + assert!(f.record().unwrap().completed && !f.journal.join("stage_1").exists()); + } + f.app.runtime.shutdown().await; + } +} +#[tokio::test] +async fn committed_plan_cannot_bypass_later_retention_or_unknown_owner_fences() { + let f = Fixture::new().await; + f.age(); + planned_without_deletion(&f).await; + *f.config.lock().unwrap() = json!({"snapshots":{"retention_days":0}}); + assert_eq!(f.reclaim().await.unwrap(), 0); + *f.config.lock().unwrap() = json!({}); + f.app + .store + .append( + "op_late_unknown", + cyber_store::Expected::Seq(-1), + vec![cyber_store::NewEvent::new( + "native.activity.changed.1", + json!({"session_id":f.session,"status":"unknown","phase":"legacy_process"}), + )], + ) + .unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(!f.record().unwrap().completed && f.journal.join("stage_0").exists()); + f.app.runtime.shutdown().await; +} +#[tokio::test] +async fn reclamation_projectors_roll_back_foreign_plans_and_invalid_completions() { + let f = Fixture::new().await; + f.age(); + planned_without_deletion(&f).await; + let plan = f.record().unwrap().plan; + let id = records::Plan::id(&plan.receipt).unwrap(); + let notifications = f.app.store.subscribe(); + let mut wrong = plan.clone(); + wrong.receipt.events_digest = "0".repeat(64); + let foreign = records::Plan::id(&wrong.receipt).unwrap(); + assert!( + f.app + .store + .append( + &foreign, + cyber_store::Expected::Seq(-1), + vec![cyber_store::NewEvent::new(records::PLANNED, json!(wrong))] + ) + .is_err() + ); + assert!(f.app.store.aggregate_seq(&foreign).unwrap().is_none()); + let completion = records::Completion { + plan_id: id.clone(), + removed_bytes: plan.total_bytes().unwrap() + 1, + }; + assert!( + f.app + .store + .append( + &id, + cyber_store::Expected::Seq(0), + vec![cyber_store::NewEvent::new( + records::COMPLETED, + json!(completion) + )] + ) + .is_err() + ); + assert!(!f.record().unwrap().completed); + assert!(notifications.try_recv().is_err()); + assert!(f.journal.join("stage_0").exists()); + f.app.runtime.shutdown().await; +} +#[tokio::test] +async fn an_oversized_reference_view_never_becomes_an_empty_cleanup_view() { + let f = Fixture::new().await; + f.age(); + let roots = (0..100001u32) + .map(|index| format!("copy:{index:064x}")) + .collect::>(); + f.app.store.append("op_retained_history",cyber_store::Expected::Seq(-1),vec![cyber_store::NewEvent::new("native.activity.changed.1",json!({"session_id":f.session,"status":"settled","phase":"historical_references","retained_refs":roots}))]).unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(f.record().is_none() && f.journal.join("stage_0").exists()); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn historical_reclamation_reprojects_after_later_fences_and_source_deletion() { + for deleted in [false, true] { + let f = Fixture::new().await; + f.age(); + assert!(f.reclaim().await.unwrap() > 0); + if deleted { + f.app.runtime.delete(&f.session).await.unwrap(); + } else { + f.app + .store + .append( + "op_later_unknown", + cyber_store::Expected::Seq(-1), + vec![cyber_store::NewEvent::new( + "native.activity.changed.1", + json!({"session_id":f.session,"status":"unknown","phase":"legacy_process"}), + )], + ) + .unwrap(); + } + let count = f + .app + .store + .reproject( + &[], + "snapshot.reclamation.", + Arc::new(|db, event| records::project(db, event)), + ) + .unwrap(); + assert_eq!(count, 2); + assert!(f.record().unwrap().completed); + assert!(!f.journal.join("stage_0").exists()); + f.app.runtime.shutdown().await; + } +} diff --git a/crates/cyber-cli/tests/support/tui_preparation_driver.py b/crates/cyber-cli/tests/support/tui_preparation_driver.py index d17b32977..b6e35123a 100644 --- a/crates/cyber-cli/tests/support/tui_preparation_driver.py +++ b/crates/cyber-cli/tests/support/tui_preparation_driver.py @@ -57,6 +57,10 @@ def forward(self): upstream.request(self.command, self.path, body, dict(self.headers)) response = upstream.getresponse() call["status"] = response.status + failure = None + if self.path.endswith("/review") and response.status != 200: + failure = response.read(65536) + call["error"] = failure.decode("utf-8", errors="replace") if self.command == "POST" and "/rewind/preparations/" in self.path and self.path.endswith("/confirm"): payload = json.loads(response.read()) assert response.status == 200, payload @@ -72,6 +76,9 @@ def forward(self): self.send_header("Connection", "close") self.end_headers() self.close_connection = True + if failure is not None: + self.wfile.write(failure) + self.wfile.flush() while block := response.read1(65536): self.wfile.write(block) self.wfile.flush() @@ -179,16 +186,22 @@ def open_preparations(terminal, proxy): def positive_review(terminal, proxy): - for attempt in range(3): + for attempt in range(20): terminal.wait(lambda: completed(proxy, "/review", success=False) > attempt, "pre-effect review response") reply = [call for call in proxy.calls if call["done"] and call["path"].endswith("/review")][-1] if reply["status"] == 200: return attempt + 1 - assert reply["status"] == 409, reply - # A busy native namespace may refuse a read-only review. Explicit Enter - # requests fresh discovery/review; no confirmation has been submitted. + error = json.loads(reply["error"]) + busy = error.get("_tag") == "SessionBusyError" or error == { + "_tag": "ConflictError", "message": "Snapshot storage is busy", + } + assert reply["status"] == 409 and busy, reply + # Rewind inspection can still own the Session while its native worker + # exits. Retry only this typed busy refusal, with bounded breathing room; + # Enter requests fresh discovery/review, never cancellation replay. + terminal.pump(0.5) terminal.send(b"\r") - raise AssertionError("native pre-effect review stayed busy") + raise AssertionError(f"native pre-effect review refused: {reply}; terminal: {terminal.text()[-3000:]!r}") def exercise(binary, session, activity, proxy): diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 797a83cf6..d3e4cbd7e 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -552,6 +552,7 @@ pub fn registry() -> EventRegistry { .map_err(|e| e.to_string()) }) .expect("valid messaging override event"); + super::snapshot_reclamation::register(&mut registry); super::branches::register(&mut registry); super::budget::register(&mut registry); super::reservations::register(&mut registry); @@ -579,6 +580,7 @@ pub fn registry() -> EventRegistry { } fn project(tx: &Transaction<'_>, e: &StoredEvent) -> Result<(), String> { + super::snapshot_reclamation::project(tx, e)?; super::deferred_tools::validate_event(tx, e)?; super::admission_authority::project(tx, e)?; super::hooks::project(tx, e)?; diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 1aec4d2af..6248dc594 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -138,6 +138,7 @@ mod snapshot_recovery_checkout; mod snapshot_recovery_types; mod snapshot_restore_receipt; pub use snapshot_lifetime::{SnapshotLifetime, SnapshotLifetimeBinding, SnapshotLifetimeRequest}; +pub mod snapshot_reclamation; mod snapshot_work; pub use rewind_preview::{ RewindAction, RewindCodePreview, RewindConfirmation, RewindEffect, RewindPreview, diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation.rs b/crates/cyber-server/src/runtime/snapshot_reclamation.rs new file mode 100644 index 000000000..ae80ee9a3 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_reclamation.rs @@ -0,0 +1,283 @@ +//! Durable private-payload cleanup plans. These never settle native activities. +use super::{SnapshotOrigin, SnapshotRestoreReceipt, activity}; +use cyber_store::{EventRegistry, StoreError, StoredEvent}; +use rusqlite::{Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; + +pub const PLANNED: &str = "snapshot.reclamation.planned.1"; +pub const COMPLETED: &str = "snapshot.reclamation.completed.1"; +const LIMIT: usize = 4096; +const BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct File { + pub name: String, + pub identity: String, + pub generation: String, + pub hash: String, + pub bytes: u64, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Plan { + pub version: u32, + pub receipt: SnapshotRestoreReceipt, + pub files: Vec, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Completion { + pub plan_id: String, + pub removed_bytes: u64, +} +#[derive(Debug, Clone)] +pub struct Record { + pub plan: Plan, + pub completed: bool, +} +fn invalid(message: impl Into) -> StoreError { + StoreError::InvalidAppend(message.into()) +} +fn hash(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} +impl Plan { + pub fn id(receipt: &SnapshotRestoreReceipt) -> Result { + let frame = serde_json::to_vec(&("snapshot-reclamation-v1", receipt)) + .map_err(|e| invalid(e.to_string()))?; + Ok(format!("snapshot_reclaim_{:x}", Sha256::digest(frame))) + } + pub fn validate(&self) -> Result<(), StoreError> { + let checkout = self + .receipt + .checkout + .as_ref() + .ok_or_else(|| invalid("Reclamation requires retained checkout evidence"))?; + self.receipt + .checkout_acknowledgement_digest() + .map_err(invalid)?; + let empty = checkout + .lifetime + .checkout_resources_digest(&[]) + .map_err(invalid)?; + if self.version != 1 + || self.receipt.origin.worktree_id.is_some() + || checkout.resources_digest != empty + || self.files.len() > LIMIT + || serde_json::to_vec(self) + .map_err(|e| invalid(e.to_string()))? + .len() + > BYTES + { + return Err(invalid( + "Reclamation plan exceeds supported ownership or bounds", + )); + } + let mut names = BTreeSet::new(); + for file in &self.files { + let index = file + .name + .strip_prefix("stage_") + .and_then(|index| index.parse::().ok()); + if index.is_none_or(|index| index >= LIMIT || file.name != format!("stage_{index}")) + || !names.insert(&file.name) + || file.identity.is_empty() + || file.identity.len() > 128 + || !hash(&file.generation) + || !hash(&file.hash) + { + return Err(invalid("Invalid private reclamation payload")); + } + } + self.total_bytes()?; + Ok(()) + } + pub fn total_bytes(&self) -> Result { + self.files.iter().try_fold(0u64, |sum, file| { + sum.checked_add(file.bytes) + .ok_or_else(|| invalid("Reclamation size overflow")) + }) + } + /// Must run at the Store writer immediately around private deletion. A caller + /// must separately retain native namespace, reader, lifetime and lease proof. + pub fn authorize(&self, db: &Connection) -> Result, StoreError> { + self.validate()?; + super::snapshot_restore_receipt::verify(db, &self.receipt)?; + let journal = &self.receipt.origin.binding.journal_id; + let directory = self + .receipt + .origin + .binding + .directory + .to_str() + .ok_or_else(|| invalid("Non-UTF8 reclamation Location"))?; + let protected: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM event e LEFT JOIN session s ON s.id=json_extract(e.data,'$.session_id') WHERE e.type='native.activity.changed.1' AND coalesce(json_extract(e.data,'$.status'),'unknown')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) AND (s.id IS NULL OR (coalesce(json_extract(e.data,'$.status'),'unknown')='unknown' AND json_extract(e.data,'$.snapshot_lifetime') IS NULL) OR s.directory=?2 OR json_extract(e.data,'$.session_id')=?3 OR json_extract(e.data,'$.snapshot_lifetime.owner.directory')=?2 OR EXISTS(SELECT 1 FROM json_tree(e.data) j WHERE j.key='journal_id' AND j.value=?1)))", + rusqlite::params![journal, directory, self.receipt.origin.session_id], |row| row.get(0), + )?; + if protected { + return Err(invalid( + "An unsettled native owner or incomplete Location view protects this journal", + )); + } + references(db) + } +} + +/// Complete bounded durable roots. Failure must never be treated as an empty set. +pub fn references(db: &Connection) -> Result, StoreError> { + let mut query = + db.prepare("SELECT DISTINCT root FROM snapshot_reference ORDER BY root LIMIT 100001")?; + let roots = query + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + if roots.len() > 100000 { + return Err(invalid( + "Snapshot reference limit exceeded; reclamation refused", + )); + } + Ok(roots) +} + +pub fn restore_receipt( + db: &Connection, + origin: &SnapshotOrigin, +) -> Result, StoreError> { + let raw: Option = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq DESC LIMIT 1", + [&origin.activity_id, activity::CHANGED], + |row| row.get(0), + ) + .optional()?; + let Some(raw) = raw else { + return Ok(None); + }; + let record: activity::Record = + serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; + let Some(receipt) = record.snapshot_restore_receipt else { + return Ok(None); + }; + if receipt.origin != *origin { + return Err(invalid("Foreign reclamation receipt")); + } + super::snapshot_restore_receipt::verify(db, &receipt)?; + Ok(Some(receipt)) +} + +pub fn lookup( + db: &Connection, + receipt: &SnapshotRestoreReceipt, +) -> Result, StoreError> { + let id = Plan::id(receipt)?; + let first: Option<(String, String)> = db + .query_row( + "SELECT type,data FROM event WHERE aggregate_id=?1 AND seq=0", + [&id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + let Some((kind, raw)) = first else { + return Ok(None); + }; + let plan: Plan = serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; + plan.validate()?; + if kind != PLANNED || plan.receipt != *receipt { + return Err(invalid("Reclamation request owner differs")); + } + let (seq, kind, raw): (i64, String, String) = db.query_row( + "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + let completed = match seq { + 0 if kind == PLANNED => false, + 1 if kind == COMPLETED => { + let completion: Completion = + serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; + if completion.plan_id != id || completion.removed_bytes != plan.total_bytes()? { + return Err(invalid("Reclamation completion differs")); + } + true + } + _ => return Err(invalid("Invalid reclamation history")), + }; + Ok(Some(Record { plan, completed })) +} + +pub(super) fn register(registry: &mut EventRegistry) { + registry + .register_with(PLANNED, |value| { + serde_json::from_value::(value.clone()) + .map_err(|e| e.to_string())? + .validate() + .map_err(|e| e.to_string()) + }) + .expect("reclamation plan event"); + registry + .register_with(COMPLETED, |value| { + serde_json::from_value::(value.clone()) + .map(|_| ()) + .map_err(|e| e.to_string()) + }) + .expect("reclamation completion event"); +} +/// Validate append ownership while allowing historical projection rebuilds after +/// later fences or Session deletion. Replaying records never grants file effects. +pub fn project(db: &Connection, event: &StoredEvent) -> Result<(), String> { + let current = if matches!(event.kind.as_str(), PLANNED | COMPLETED) { + db.query_row( + "SELECT seq FROM event_sequence WHERE aggregate_id=?1", + [&event.aggregate_id], + |row| row.get::<_, i64>(0), + ) + .optional() + .map_err(|error| error.to_string())? + .unwrap_or(-1) + } else { + return Ok(()); + }; + let appending = current < event.seq; + match event.kind.as_str() { + PLANNED => { + let plan: Plan = + serde_json::from_value(event.data.clone()).map_err(|e| e.to_string())?; + if event.seq != 0 + || Plan::id(&plan.receipt).map_err(|e| e.to_string())? != event.aggregate_id + { + return Err("Foreign reclamation plan owner".into()); + } + plan.validate().map_err(|e| e.to_string())?; + if appending { + plan.authorize(db).map_err(|e| e.to_string())?; + } + } + COMPLETED => { + let completion: Completion = + serde_json::from_value(event.data.clone()).map_err(|e| e.to_string())?; + let raw: String = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=0 AND type=?2", + [&event.aggregate_id, PLANNED], + |row| row.get(0), + ) + .map_err(|e| e.to_string())?; + let plan: Plan = serde_json::from_str(&raw).map_err(|e| e.to_string())?; + if event.seq != 1 + || completion.plan_id != event.aggregate_id + || completion.removed_bytes != plan.total_bytes().map_err(|e| e.to_string())? + { + return Err("Foreign reclamation completion".into()); + } + plan.validate().map_err(|e| e.to_string())?; + if appending { + plan.authorize(db).map_err(|e| e.to_string())?; + } + } + _ => {} + } + Ok(()) +} diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index 5711ffb0f..6ea97f97d 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -218,6 +218,26 @@ impl Store { phase.checkout_bindings = Some(resources.to_vec()); self.write_source_phase(binding, &phase) } + pub(crate) fn verify_reclamation_phase( + &self, + binding: &SnapshotLifetimeBinding, + origin: &cyber_server::runtime::SnapshotOrigin, + resources_digest: &str, + ) -> Result<(), String> { + let phase = self.read_source_phase(binding)?; + let resources = phase + .checkout_bindings + .as_ref() + .ok_or("Missing acquired reclamation resources")?; + if !phase.admitted + || phase.origin.as_ref() != Some(origin) + || !resources.is_empty() + || binding.checkout_resources_digest(resources)? != resources_digest + { + return Err("Reclamation source phase contradicts committed restore evidence".into()); + } + Ok(()) + } fn write_source_phase( &self, binding: &SnapshotLifetimeBinding, diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 7e2905748..0c69a9ed5 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -12,6 +12,7 @@ mod inventory; mod links; mod ownership; mod preparation; +pub(crate) mod reclamation; mod resolution; mod restore; mod storage; diff --git a/crates/cyber-snapshot/src/copy/ownership.rs b/crates/cyber-snapshot/src/copy/ownership.rs index 4b13b29c6..d99acfc4f 100644 --- a/crates/cyber-snapshot/src/copy/ownership.rs +++ b/crates/cyber-snapshot/src/copy/ownership.rs @@ -482,6 +482,11 @@ impl Ownership { store.verify_root()?; Ok(Observations { files, parents }) } + pub(super) fn original_identity(&self, index: usize) -> Option<&str> { + self.files + .get(index) + .and_then(|file| file.source.as_deref()) + } pub(super) fn verify_original(&self, index: usize, file: &std::fs::File) -> Result<()> { if self.files.get(index).and_then(|file| file.source.as_ref()) != Some(&storage::identity(file)?) diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs new file mode 100644 index 000000000..150e5305e --- /dev/null +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -0,0 +1,419 @@ +//! Reclaim only exact acknowledged displaced images; keep intent/lifetime receipts. +use super::{ + Store, ownership, + restore::{Journal, JournalBackend}, + storage, +}; +use crate::reclamation::{Policy, Settlement}; +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; +use cyber_server::runtime::{ + SnapshotLifetime, + snapshot_reclamation::{File as Payload, Plan}, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, + sync::Arc, +}; + +struct Prepared { + store: Store, + journal: Dir, + intent: MetadataPin, + phase: MetadataPin, + workers: Dir, + plan: Plan, + lifetime: super::lifetime::Lifetime, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + settled_at_ms: i64, +} +struct MetadataPin { + file: std::fs::File, + identity: String, + generation: String, +} +impl MetadataPin { + fn open(dir: &Dir, name: &str) -> Result { + let file = storage::open(dir, Path::new(name))?; + storage::validate_lock(&file)?; + Ok(Self { + identity: storage::identity(&file)?, + generation: ownership::generation(&file)?, + file, + }) + } + fn verify(&self, dir: &Dir, name: &str) -> Result<(), String> { + let route = storage::open(dir, Path::new(name))?; + storage::validate_lock(&route)?; + if storage::identity(&route)? != self.identity + || storage::identity(&self.file)? != self.identity + || ownership::generation(&route)? != self.generation + || ownership::generation(&self.file)? != self.generation + { + return Err("Reclamation metadata inode or generation changed".into()); + } + Ok(()) + } +} +fn due(config: &crate::ConfigFn, root: &Path, settled_at_ms: i64) -> bool { + let value = config(root); + if value.get("snapshots") == Some(&serde_json::Value::Bool(false)) { + return false; + } + let days = value + .pointer("/snapshots/retention_days") + .and_then(serde_json::Value::as_u64) + .unwrap_or(7); + let elapsed = (super::now() as u128) + .saturating_mul(1000) + .saturating_sub(settled_at_ms.max(0) as u128); + settled_at_ms > 0 && days != 0 && elapsed > u128::from(days).saturating_mul(86400000) +} +fn ordinary(record: &Journal) -> bool { + record.version >= 3 + && record.completed + && record.acknowledged + && record.origin.is_some() + && record.resolution.is_none() + && record.settled_by.is_none() + && record + .context + .as_ref() + .is_some_and(|context| context.git.is_none()) + && record.backend != Some(JournalBackend::Git) + && record + .changes + .iter() + .any(|change| change.before.as_ref().is_some_and(|entry| entry.length > 0)) +} +/// Read existing metadata only. A missing/partial namespace remains protected. +pub(crate) fn locations(data: &Path) -> Result, String> { + if !data.exists() { + return Ok(BTreeSet::new()); + } + let data = Dir::open_ambient_dir(data, cap_std::ambient_authority()).map_err(storage::fail)?; + let snapshot = match data.symlink_metadata("snapshot") { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeSet::new()), + _ => storage::existing_child(&data, "snapshot", false)?, + }; + let root = match snapshot.symlink_metadata("global") { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeSet::new()), + _ => storage::existing_child(&snapshot, "global", true)?, + }; + let mut result = BTreeSet::new(); + let mut count = 0usize; + for name in storage::names(&root, 100000)? { + if !storage::valid_digest(&name) { + return Err("Unknown snapshot namespace; reclamation refused".into()); + } + let namespace = storage::existing_child(&root, &name, true)?; + let journals = match namespace.symlink_metadata("journals") { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + _ => storage::existing_child(&namespace, "journals", true)?, + }; + for name in storage::names(&journals, super::ENTRY_LIMIT)? { + count += 1; + if count > 100000 { + return Err("Snapshot reclamation inventory limit exceeded".into()); + } + let directory = journals.open_dir_nofollow(&name).map_err(storage::fail)?; + let record: Journal = match directory.symlink_metadata("intent.json") { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + _ => serde_json::from_slice(&storage::read( + &directory, + "intent.json", + super::RECORD_LIMIT, + )?) + .map_err(storage::fail)?, + }; + if ordinary(&record) { + result.insert( + record + .location + .to_str() + .ok_or("Non-UTF8 snapshot Location")? + .to_owned(), + ); + } + } + } + Ok(result) +} + +pub(crate) fn reclaim( + store: Store, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + policy: Arc, + id: &str, +) -> Result { + if store.preparation_cancellation(id)?.is_some() { + return Ok(0); + } + let (directory, record) = store.journal(id)?; + if !ordinary(&record) { + return Ok(0); + } + let origin = record.origin.as_ref().ok_or("Missing reclamation origin")?; + let Some(settlement) = policy.receipt(origin)? else { + return Ok(0); + }; + if !due(config.as_ref(), &store.root, settlement.settled_at_ms) { + return Ok(0); + } + let existing = policy.record(&settlement.receipt)?; + if existing.as_ref().is_some_and(|record| record.completed) { + return Ok(0); + } + let frozen = existing.map(|record| record.plan); + let prepared = prepare( + store, directory, record, settlement, frozen, workspace, config, + )?; + let plan = prepared.plan.clone(); + if plan.files.is_empty() { + return Ok(0); + } + policy.reclaim(plan, Box::new(move || prepared.delete())) +} +impl Store { + pub(crate) fn reclamation_journals(&self) -> Result, String> { + storage::names(&self.journals, super::ENTRY_LIMIT) + } +} + +fn prepare( + store: Store, + journal: Dir, + record: Journal, + settlement: Settlement, + frozen: Option, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, +) -> Result { + let receipt = settlement.receipt; + let origin = &receipt.origin; + store.verify_worker_namespace(origin)?; + if store.worker_digest(Some(origin))?.as_ref() != Some(&receipt.intent_digest) { + return Err("Reclamation intent differs from committed restore".into()); + } + let checkout = receipt + .checkout + .as_ref() + .ok_or("Legacy restore has no cleanup checkout proof")?; + let lifetime = store.lock_lifetime(&checkout.lifetime)?; + let resources = lifetime + .checkout_bindings()? + .ok_or("Missing acquired checkout inventory")?; + if !resources.is_empty() + || checkout.resources_digest != checkout.lifetime.checkout_resources_digest(&resources)? + { + return Err("Managed checkout reclamation requires retained resource guards".into()); + } + unmanaged(&store)?; + let ownership = record + .ownership + .as_ref() + .ok_or("Legacy restore has no payload ownership")?; + let mut files = Vec::new(); + let existing = frozen.as_ref().map(|plan| { + plan.files + .iter() + .map(|file| (file.name.as_str(), file)) + .collect::>() + }); + let mut known = BTreeSet::from(["intent.json".to_owned()]); + for (index, change) in record.changes.iter().enumerate() { + let name = format!("stage_{index}"); + known.insert(name.clone()); + let observed = + ownership::observe_image(&store, Some(&journal), std::ffi::OsStr::new(&name))?; + let expected = existing + .as_ref() + .and_then(|files| files.get(name.as_str()).copied()); + match (change.before.as_ref(), observed) { + (Some(before), Some(image)) + if before.kind == super::ImageKind::File && image.entry == *before => + { + let pin = storage::open(&journal, Path::new(&name))?; + ownership.verify_original(index, &pin)?; + let file = Payload { + name: name.clone(), + identity: image.identity, + generation: image.generation, + hash: image.entry.object, + bytes: image.entry.length, + }; + if existing.is_some() && expected != Some(&file) { + return Err("Planned payload changed before reclamation".into()); + } + files.push(file); + } + (Some(before), None) + if expected.is_some_and(|file| { + file.hash == before.object + && file.bytes == before.length + && ownership.original_identity(index) == Some(file.identity.as_str()) + }) => + { + files.push(expected.cloned().ok_or("Missing frozen payload")?) + } + (None, None) if expected.is_none() => {} + _ => return Err("Displaced payload lacks exact original ownership".into()), + } + } + for name in storage::names(&journal, super::ENTRY_LIMIT * 3)? { + if !known.contains(&name) { + return Err("Unknown journal file or construction stage; reclamation refused".into()); + } + } + let plan = Plan { + version: 1, + receipt, + files, + }; + plan.validate().map_err(storage::fail)?; + if frozen.as_ref().is_some_and(|frozen| frozen != &plan) { + return Err("Reclamation plan differs from retained tombstone".into()); + } + let intent = MetadataPin::open(&journal, "intent.json")?; + let workers = storage::existing_child(&store.namespace, "workers", true)?; + let binding = lifetime.binding(); + let name = binding + .source_phase + .as_deref() + .ok_or("Missing reclamation source phase")?; + let phase = MetadataPin::open(&workers, name)?; + store.verify_reclamation_phase( + binding, + &plan.receipt.origin, + &plan + .receipt + .checkout + .as_ref() + .ok_or("Missing reclamation checkout")? + .resources_digest, + )?; + phase.verify(&workers, name)?; + storage::existing_child( + &store.journals, + &plan.receipt.origin.binding.journal_id, + true, + )?; + Ok(Prepared { + store, + journal, + intent, + phase, + workers, + plan, + lifetime, + workspace, + config, + settled_at_ms: settlement.settled_at_ms, + }) +} +fn unmanaged(store: &Store) -> Result<(), String> { + if !cyber_core::worktrees::Repository::managed_locations_at(&store.root) + .map_err(storage::fail)? + .is_empty() + { + return Err("Snapshot cleanup checkout inventory changed".into()); + } + Ok(()) +} +impl Prepared { + fn verify_routes(&self) -> Result<(), String> { + let _reader_lease = &self.workspace; + self.lifetime.verify()?; + self.store + .verify_worker_namespace(&self.plan.receipt.origin)?; + if !due(self.config.as_ref(), &self.store.root, self.settled_at_ms) { + return Err("Snapshot retention no longer permits reclamation".into()); + } + self.intent.verify(&self.journal, "intent.json")?; + self.phase.verify( + &self.workers, + self.lifetime + .binding() + .source_phase + .as_deref() + .ok_or("Missing source phase route")?, + )?; + unmanaged(&self.store) + } + fn verify(&self) -> Result<(), String> { + self.verify_routes()?; + let (_, record) = self + .store + .journal(&self.plan.receipt.origin.binding.journal_id)?; + if !ordinary(&record) + || self + .store + .worker_digest(Some(&self.plan.receipt.origin))? + .as_ref() + != Some(&self.plan.receipt.intent_digest) + { + return Err("Acknowledged reclamation intent changed".into()); + } + let names = self + .plan + .files + .iter() + .map(|file| file.name.as_str()) + .chain(std::iter::once("intent.json")) + .collect::>(); + if storage::names(&self.journal, super::ENTRY_LIMIT * 3)? + .iter() + .any(|name| !names.contains(name.as_str())) + { + return Err("Unknown journal entry appeared before reclamation".into()); + } + unmanaged(&self.store) + } + fn delete(self) -> Result { + self.verify()?; + for file in &self.plan.files { + self.verify_file(file)?; + } + let directory = self + .journal + .try_clone() + .map_err(storage::fail)? + .into_std_file(); + let mut generation = ownership::generation(&directory)?; + for file in &self.plan.files { + self.verify_routes()?; + if ownership::generation(&directory)? != generation { + return Err("Reclamation journal directory changed during deletion".into()); + } + if let Some(_pin) = self.verify_file(file)? { + self.journal + .remove_file(&file.name) + .map_err(storage::fail)?; + storage::sync(&self.journal)?; + generation = ownership::generation(&directory)?; + } + } + // Full inventory/receipt checks surround the bounded per-leaf route + // checks, avoiding a full intent decode for every displaced image. + self.verify()?; + storage::sync(&self.journal)?; + self.plan.total_bytes().map_err(storage::fail) + } + fn verify_file(&self, file: &Payload) -> Result, String> { + match self.journal.symlink_metadata(&file.name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(storage::fail(error)), + _ => {} + } + let pin = storage::open(&self.journal, Path::new(&file.name))?; + if storage::identity(&pin)? != file.identity + || ownership::generation(&pin)? != file.generation + { + return Err("Reclamation payload identity or metadata changed".into()); + } + Ok(Some(pin)) + } +} diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 75c7011e2..d0bcc2304 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -8,6 +8,7 @@ #[cfg(any(target_os = "macos", target_os = "linux"))] mod copy; mod git; +pub mod reclamation; mod restore; pub use cyber_server::runtime::SnapshotRecovery; @@ -45,6 +46,7 @@ pub struct GitSnapshots { private_roots: Vec, references: Option>, unavailable: Mutex>, + reclamation: Option>, } impl GitSnapshots { @@ -58,8 +60,19 @@ impl GitSnapshots { config: Arc, private_roots: Vec, references: Option>, + ) -> Arc { + Self::with_reclamation(data_dir, config, private_roots, references, None) + } + + pub fn with_reclamation( + data_dir: PathBuf, + config: Arc, + private_roots: Vec, + references: Option>, + reclamation: Option>, ) -> Arc { Arc::new(Self { + reclamation, data_dir, config, private_roots, @@ -327,17 +340,141 @@ impl GitSnapshots { }) } - pub async fn gc(&self) { + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn track_with_reclamation(&self, directory: &str) -> Result { + let result = self + .file_copy(directory.to_owned(), |store| store.track()) + .await; + let Some(policy) = &self.reclamation else { + return result; + }; + if !result + .as_ref() + .is_err_and(|error| error.starts_with("Non-Git snapshot storage cap reached")) + { + return result; + } + match self.reclaim(directory).await { + Ok(bytes) if bytes > 0 => { + self.file_copy(directory.to_owned(), |store| store.track()) + .await + } + Ok(_) => result, + Err(error) => { + policy.refused(directory, &error); + result + } + } + } + + /// Reclaim one independently settled ordinary journal. Intent/lifetime + /// metadata and durable retry receipts remain retained. No source effects. + pub async fn reclaim(&self, directory: &str) -> Result { #[cfg(any(target_os = "macos", target_os = "linux"))] - if let Some(references) = &self.references - && let Ok(protected) = references() { + let policy = self + .reclamation + .clone() + .ok_or("Snapshot reclamation authority is unavailable")?; + self.recovery_location(directory)?; + let directory = directory.to_owned(); let data = self.data_dir.clone(); - let config = self.config.clone(); - let _ = tokio::task::spawn_blocking(move || copy::gc(&data, &config, &protected)).await; + let options = self.recovery_options(&directory); + let lock = cyber_server::workspace::mutation_lock(Path::new(&directory)) + .map_err(|error| error.to_string())?; + let reader = lock + .clone() + .try_lock_owned() + .map_err(|_| "Snapshot reader or writer is active")?; + let (source, native_data, native_options) = + (directory.clone(), data.clone(), options.clone()); + let journals = tokio::task::spawn_blocking(move || { + let _reader = reader; + copy::Store::recover_existing(&native_data, &source, native_options)? + .reclamation_journals() + }) + .await + .map_err(|error| error.to_string())??; + let mut refusal = None; + for journal in journals { + let workspace = lock + .clone() + .try_lock_owned() + .map_err(|_| "Snapshot reader or writer is active")?; + let (source, native_data, native_options, native_policy, config) = ( + directory.clone(), + data.clone(), + options.clone(), + policy.clone(), + self.config.clone(), + ); + let result = tokio::task::spawn_blocking(move || { + let store = + copy::Store::recover_existing(&native_data, &source, native_options)?; + copy::reclamation::reclaim(store, workspace, config, native_policy, &journal) + }) + .await + .map_err(|error| error.to_string())?; + match result { + Ok(0) => {} + Ok(bytes) => return Ok(bytes), + Err(error) => { + refusal.get_or_insert(error); + } + } + } + refusal.map_or(Ok(0), Err) + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = directory; + Err("Native snapshot reclamation is unsupported on this host".into()) + } + } + + pub async fn gc(&self) { + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + if let Some(policy) = &self.reclamation { + let data = self.data_dir.clone(); + let locations = + tokio::task::spawn_blocking(move || copy::reclamation::locations(&data)).await; + match locations { + Ok(Ok(locations)) => { + for directory in locations { + if let Err(error) = self.reclaim(&directory).await { + policy.refused(&directory, &error); + } + } + } + Ok(Err(error)) => policy.refused("snapshot inventory", &error), + Err(error) => policy.refused("snapshot inventory", &error.to_string()), + } + let (data, config, policy) = + (self.data_dir.clone(), self.config.clone(), policy.clone()); + let report = policy.clone(); + let result = tokio::task::spawn_blocking(move || { + policy.collect(Box::new(move |protected| { + copy::gc(&data, &config, protected) + })) + }) + .await; + if let Err(error) = result + .map_err(|error| error.to_string()) + .and_then(|value| value) + { + report.refused("snapshot collection", &error); + } + } else if let Some(references) = &self.references + && let Ok(protected) = references() + { + let (data, config) = (self.data_dir.clone(), self.config.clone()); + let _ = + tokio::task::spawn_blocking(move || copy::gc(&data, &config, &protected)).await; + } } #[cfg(not(any(target_os = "macos", target_os = "linux")))] - let _ = &self.references; + let _ = (&self.references, &self.reclamation); for git_dir in git::shadow_dirs(&self.root()) { git::gc(&git_dir).await; } @@ -633,9 +770,7 @@ impl Snapshots for GitSnapshots { if self.non_git(&directory) { #[cfg(any(target_os = "macos", target_os = "linux"))] { - let result = self - .file_copy(directory.clone(), |store| store.track()) - .await; + let result = self.track_with_reclamation(&directory).await; let mut unavailable = self .unavailable .lock() diff --git a/crates/cyber-snapshot/src/reclamation.rs b/crates/cyber-snapshot/src/reclamation.rs new file mode 100644 index 000000000..82146ceba --- /dev/null +++ b/crates/cyber-snapshot/src/reclamation.rs @@ -0,0 +1,25 @@ +//! Application-owned Store authority for private snapshot maintenance. +use cyber_server::runtime::{ + SnapshotOrigin, SnapshotRestoreReceipt, + snapshot_reclamation::{Plan, Record}, +}; +use std::collections::BTreeSet; + +#[derive(Debug, Clone)] +pub struct Settlement { + pub receipt: SnapshotRestoreReceipt, + pub settled_at_ms: i64, +} + +pub type Delete = Box Result + Send>; +pub type Collect = Box) -> Result<(), String> + Send>; +/// Implementations serialize verification/deletion against reference writers. +/// A durable plan must be committed before invoking `delete`; native guards are +/// retained by that closure. Neither method may acknowledge a native activity. +pub trait Policy: Send + Sync { + fn receipt(&self, origin: &SnapshotOrigin) -> Result, String>; + fn record(&self, receipt: &SnapshotRestoreReceipt) -> Result, String>; + fn reclaim(&self, plan: Plan, delete: Delete) -> Result; + fn collect(&self, collect: Collect) -> Result<(), String>; + fn refused(&self, directory: &str, error: &str); +} diff --git a/crates/cyber-store/migrations/20261107030000_snapshot_reference_index.sql b/crates/cyber-store/migrations/20261107030000_snapshot_reference_index.sql new file mode 100644 index 000000000..32d6b13c1 --- /dev/null +++ b/crates/cyber-store/migrations/20261107030000_snapshot_reference_index.sql @@ -0,0 +1,28 @@ +-- A derived, event-owned index. Keep bookkeeping in the event transaction even +-- for imports or metadata repairs that do not use the runtime's projectors. +CREATE TABLE snapshot_reference ( + event_id TEXT NOT NULL REFERENCES event(id) ON DELETE CASCADE, + root TEXT NOT NULL CHECK ( + length(root)=69 AND substr(root,1,5)='copy:' + AND substr(root,6) NOT GLOB '*[^0-9a-f]*' + ), + PRIMARY KEY(event_id,root) +) STRICT; +CREATE INDEX snapshot_reference_root ON snapshot_reference(root); +INSERT INTO snapshot_reference(event_id,root) +SELECT DISTINCT e.id,j.atom FROM event e,json_tree(e.data) j +WHERE j.type='text' AND length(j.atom)=69 AND substr(j.atom,1,5)='copy:' +AND substr(j.atom,6) NOT GLOB '*[^0-9a-f]*'; +CREATE TRIGGER snapshot_reference_insert AFTER INSERT ON event BEGIN + INSERT OR IGNORE INTO snapshot_reference(event_id,root) + SELECT NEW.id,j.atom FROM json_tree(NEW.data) j + WHERE j.type='text' AND length(j.atom)=69 AND substr(j.atom,1,5)='copy:' + AND substr(j.atom,6) NOT GLOB '*[^0-9a-f]*'; +END; +CREATE TRIGGER snapshot_reference_update AFTER UPDATE OF data ON event BEGIN + DELETE FROM snapshot_reference WHERE event_id=NEW.id; + INSERT OR IGNORE INTO snapshot_reference(event_id,root) + SELECT NEW.id,j.atom FROM json_tree(NEW.data) j + WHERE j.type='text' AND length(j.atom)=69 AND substr(j.atom,1,5)='copy:' + AND substr(j.atom,6) NOT GLOB '*[^0-9a-f]*'; +END; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index 77c87e1e9..7ec29ee18 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -201,6 +201,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261107020000_snapshot_checkout_acknowledgements", sql: include_str!("../migrations/20261107020000_snapshot_checkout_acknowledgements.sql"), }, + Migration { + id: "20261107030000_snapshot_reference_index", + sql: include_str!("../migrations/20261107030000_snapshot_reference_index.sql"), + }, ]; pub(crate) enum Outcome { @@ -313,7 +317,7 @@ mod tests { ) .unwrap(); assert!(matches!( - migrate(&mut conn, MIGRATIONS).unwrap(), + migrate(&mut conn, &MIGRATIONS[..=at]).unwrap(), Outcome::Current )); assert_eq!( @@ -332,7 +336,7 @@ mod tests { }; assert_eq!( unknown, - MIGRATIONS[at..] + MIGRATIONS[at..=at] .iter() .map(|migration| migration.id) .collect::>() @@ -544,7 +548,7 @@ mod tests { }) .unwrap(); migrate(&mut db, &MIGRATIONS[..at]).unwrap(); - migrate(&mut db, MIGRATIONS).unwrap(); + migrate(&mut db, &MIGRATIONS[..=at]).unwrap(); let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { panic!("previous writer accepted native checkout disposition"); }; @@ -559,6 +563,92 @@ mod tests { ); } + #[test] + fn snapshot_reference_index_backfills_and_tracks_event_transactions() { + let mut db = Connection::open_in_memory().unwrap(); + db.execute_batch("PRAGMA foreign_keys=ON").unwrap(); + let at = MIGRATIONS + .iter() + .position(|migration| migration.id == "20261107030000_snapshot_reference_index") + .unwrap(); + migrate(&mut db, &MIGRATIONS[..at]).unwrap(); + let first = format!("copy:{}", "a".repeat(64)); + let second = format!("copy:{}", "b".repeat(64)); + let evidence = serde_json::json!({ + "nested": [{"snapshot": first}, first], + "invalid": ["copy:short", format!("copy:{}", "A".repeat(64)), + format!("copy:{}", "g".repeat(64)), format!("copy:{}", "a".repeat(65))] + }) + .to_string(); + db.execute( + "INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('old','ses_old',0,'retained.1',?1,1)", + [&evidence], + ).unwrap(); + migrate(&mut db, MIGRATIONS).unwrap(); + let roots = |db: &Connection| -> Vec<(String, String)> { + db.prepare("SELECT event_id,root FROM snapshot_reference ORDER BY event_id,root") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap() + }; + assert_eq!(roots(&db), [("old".into(), first.clone())]); + assert_eq!( + db.query_row("SELECT data FROM event WHERE id='old'", [], |row| row + .get::<_, String>(0)) + .unwrap(), + evidence + ); + + // Generic imports retain shared roots independently of runtime projectors. + let fresh = serde_json::json!([first, second, second]).to_string(); + db.execute( + "INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('new','ses_new',0,'imported.1',?1,2)", + [&fresh], + ).unwrap(); + assert_eq!( + roots(&db), + [ + ("new".into(), first.clone()), + ("new".into(), second.clone()), + ("old".into(), first.clone()) + ] + ); + // A failed append/projector transaction must not publish new references. + { + let tx = db.transaction().unwrap(); + tx.execute( + "INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('rolled_back','ses_failed',0,'imported.1',?1,3)", + [&fresh], + ).unwrap(); + assert_eq!(roots(&tx).len(), 5); + tx.rollback().unwrap(); + } + assert_eq!(roots(&db).len(), 3); + db.execute( + "UPDATE event SET data=?1 WHERE id='old'", + [serde_json::json!({"repaired":second}).to_string()], + ) + .unwrap(); + assert_eq!( + roots(&db), + [ + ("new".into(), first), + ("new".into(), second.clone()), + ("old".into(), second.clone()) + ] + ); + db.execute("DELETE FROM event WHERE id='new'", []).unwrap(); + assert_eq!(roots(&db), [("old".into(), second)]); + db.execute("DELETE FROM event WHERE id='old'", []).unwrap(); + assert!(roots(&db).is_empty()); + let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { + panic!("previous writer accepted reference-index storage"); + }; + assert_eq!(unknown, [MIGRATIONS[at].id]); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/docs/rewind.md b/docs/rewind.md index db254e5d6..483af12ff 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -164,8 +164,20 @@ A cap warning is emitted once per Location/condition in the running service; conversation work and successful `exec` results continue. A fresh capture retries availability after conditions change. Durable Session references protect staged baselines and inactive branches. Unreferenced old manifests/objects can be pruned -only with a complete durable reference view. Every namespace containing journals -is conservatively retained; no quota or timeout discards unresolved evidence. +only with a complete durable reference view. Namespaces containing journals remain +protected from manifest/object pruning; no quota or timeout discards unresolved +evidence. + +On admitted Linux/macOS hosts, periodic maintenance can reclaim displaced regular +file payloads from an ordinary, completed and natively acknowledged file-copy +restore in an unmanaged Location. It requires the exact durable restore receipt, +exited worker proof, a complete reference view and the retention period measured +from durable settlement. Disabled snapshots or zero retention preserve payloads. +Live readers, independent unknown effects, changed payloads or incomplete native +ownership refuse cleanup. A full capture may attempt the same eligible cleanup +and retry once. Cleanup retains the intent, lifetime records and durable retry +receipt, and never reads or restores source files. Managed Git, keep/rollback, +preparation-construction and legacy payload reclamation remain unfinished. Before file changes, restore durably publishes an intent and prepares private stages. Replacements retain displaced inodes; progress and completion are flushed diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index fbf6a5b2c..f77f3fc36 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3141,3 +3141,127 @@ library available (it runs by default in Linux/macOS branch integration tests): ```sh CARGO_INCREMENTAL=0 cargo test --locked -p cyber-cli --test branches_cli terminal_preparation_recovery -- --test-threads=1 ``` + +## Ordinary acknowledged file-copy payload reclamation (2026-11-07, local) + +The first M6 cleanup path reclaims displaced regular-file stages from completed, +natively acknowledged ordinary file-copy restores in an unmanaged Location. It +requires an immutable ordinary restore receipt, version-2 exited worker lifetime, +positively published empty checkout resources and unchanged native source phase. +It does not admit managed Git, keep/rollback resolution, cancelled preparation, +partial construction or legacy payload cleanup. M6 and parent tasks remain open. + +A deterministic receipt-owned Store aggregate records a bounded immutable plan +before deletion. It freezes each stage leaf's original inode identity, metadata +generation, BLAKE3 identity and size. The application adapter re-attests the receipt +and complete reference view in its serialized writer transaction while the native +callback retains namespace, Location reader/mutation and lifetime locks. Native +intent/phase pins, current unmanaged checkout inventory, fresh retention policy +and every remaining leaf are checked. Missing leaves count as resumable progress +only for an already committed matching plan. No source image is read or installed; +intents, worker metadata, ordinary receipts and all referenced CAS images remain. +A separate completion event preserves cumulative accounting and idempotent retry. +A cap failure can invoke the same admission and retry capture once. Existing +fallback collection now reads references under the writer that excludes concurrent +reference commits. Its conservative worker/journal gates remain in place; complete +CAS/metadata/worker reclamation is still required. + +The reference index migration backfills canonical copy manifest identities from +retained event values, then uses transactional SQLite insert/update triggers and +per-event deletion cascades. It avoids reparsing all conversations at every cleanup +admission. The old writer catalog refuses the new migration. Migration regressions +cover nested/duplicate/shared roots, invalid identities, original event byte +preservation, generic imports, transaction rollback, metadata repair, deletion and +older-catalog refusal. Two older compatibility tests initially failed because they +applied the entire later catalog while asserting their original marker's schema or +single unknown ID. They now test that exact marker migration; the separate index +case tests the new schema and older-catalog refusal. + +Real-App integration uses actual scripted-provider prompts, file-copy snapshots, +reviewed rewind, paired Store settlement and native acknowledgement. It verifies +reclaimed stages, unchanged retained images/intent, later source symlink/target, +unchanged transcript/inference, receipt reconciliation, repeated cleanup, retention +age/zero/disabled capture, real reader/lifetime locks, independent unknown/missing +Location owners, aliases/replacements/missing leaves/unknown entries, contradictory +native source phase, cap-pressure retry and complete-reference bounds. Pending +plans still obey later retention changes and unknown-owner fences. Historical +projection replay remains possible after later uncertainty or Session deletion; +replaying records grants no filesystem or native acknowledgement authority. + +Two actual process-death boundaries launch a separate test executable using the +same production App and native cleanup adapters. A test-only Policy wrapper pauses +inside the consuming writer callback before native deletion or after all private +deletions/fsyncs but before callback return/completion append. The parent observes +the boundary, sends SIGKILL, verifies signal 9, reaps the process, reopens the actual +file-backed Store and resumes through the normal policy. Both retain a pending +durable plan and complete idempotently without replaying source, including a later +owner's symlink and target. The ordinary suite helper is inert without its explicit +fixture environment. A separate partial-unlink state test is synthetic and is not +claimed as a process kill between individual unlinks. Native writer commit, +every-unlink, metadata replacement and all other crash/platform cuts remain open. + +Code-review skill inspection covered all changed production/test/migration files: +Store/native ownership separation, reference atomicity, bounded plans and names, +receipt identity, native lock lifetime and no-follow private deletion, replay versus +new admission, retention and quota semantics, error/evidence preservation, process +cleanup and platform cfgs. Review corrected an inconsistent multi-query reader by +using explicit read transactions for maintenance receipt/plan lookup, and added +native phase contradiction checks and metadata pins before accepting deletion. +Rust uses the workspace Clippy cognitive-complexity threshold (15), without +allowances; the cognitive-complexity skill has no Rust analyzer. The complete M6 +feature still needs managed resources, resolution/preparation construction classes, +CAS/metadata reclamation and the full native class/lease/platform/crash matrix. + +Verification on local macOS arm64: the full Store suite passed (65 tests across +its unit/integration suites); the final App cleanup invocation passed all 14 cases +in 46.76 seconds, including its inert child-process helper. The subsequently added +periodic-maintenance entry-point regression passed separately in 4.80 seconds; +the combined 15-case suite was not rerun after that test-only addition. HTTP passed +all 80 cases, including the checked OpenAPI document. All-target Store/App/snapshot/ +server Clippy passed after the production review corrections; App's final test +target also passed Clippy after the periodic case was added. Generated SDK +consistency, format/diff checks, strict OpenSpec (117 passed) and spec lint (0 +errors/25 warnings) passed. Earlier native snapshot verification for this cleanup +implementation passed 163 tests with 5 ignored, before the final reference-index +migration and maintenance reader-transaction correction; that broader suite was +not repeated for those later Store/App changes. C sandbox deprecations and a linker +oversized-unwind warning were observed without unchanged-main origin attribution. +All Cargo test/Clippy commands used CARGO_INCREMENTAL=0 and --locked. No Linux, +Windows, macOS Intel, source-bound CI, full workspace or full SDK test invocation +ran for this cleanup increment. No goal task/issue is closed by these local checks. + +A final incomplete-view audit corrected SQL NULL comparison semantics: a retained +native activity lacking its status field now protects cleanup as unknown rather +than falling out of the eligibility query. The expanded four-case independent +owner regression passed in 24.13 seconds and verifies unchanged retained event +bytes. All-target Store/App/snapshot/server Clippy passed again after this fix. +The full serial CLI branch/recovery suite passed 17 cases but failed its real PTY +preparation case after three HTTP 409 review responses; the driver previously +labelled every such response as busy without retaining its error. Investigation +now preserves bounded review error bodies and terminal diagnostics without logging +authorization headers. This failure is not attributed to unchanged main and the +cleanup increment was not represented as passing full CLI acceptance at that point. + +With review-body diagnostics enabled, the focused real-terminal case passed both +boundaries in 79.89 seconds. This does not establish the cause or resolve the full +invocation's intermittent refusal. The driver now retries only a typed +SessionBusyError, allowing a bounded half-second pause between at most 20 fresh +read-only reviews while native inspection tears down. Any other refusal fails +with its bounded error/terminal evidence; mutation and exactly-once checks are +unchanged. Python compilation and the cognitive-complexity skill pass (maximum +13, complexipy 8.3.0, no analyzer warnings). The next full run again passed 17 cases and failed the PTY case in 244.84 +seconds; its captured body identified the exact refusal as ConflictError with +message "Snapshot storage is busy". The bounded read-only retry now additionally +recognizes only that exact native-lock error; other conflicts still fail. This +is test synchronization, with production admission and mutation assertions +unchanged. The final complete serial invocation passed all 18 cases in 287.58 +seconds, including both terminal process-death boundaries, with that correction. +The earlier failures remain recorded rather than attributed to unchanged main. + +Review result for this ordinary-payload increment: no unresolved critical/major +code finding remains after the receipt-read, native-phase, incomplete-owner-view +and test-synchronization corrections. Existing snapshot ports and the application +Store adapter retain their project-specific dependency roles. The full recovery +feature still requires the acceptance gaps above; this is not approval of the +complete M6 feature or either issue. All final local checks are recorded above; +no PR/push or parent-task completion is claimed. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 9f0b055b7..2fbfff436 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -787,3 +787,44 @@ retain a request ID before confirmation, use lookup after reply loss, and keep native reconciliation explicit. Stale/foreign replies and missing current proof SHALL NOT enable cancellation; closing/navigation SHALL preserve drafts, attachments and all independent uncertainty. + +### Ordinary file-copy payload reclamation + +The initial reclamation admission covers completed, natively acknowledged ordinary +file-copy restores with an exact immutable Store receipt, version-2 lifetime and a +positively persisted empty checkout resource set in a currently unmanaged Location. +Managed checkout, resolution, preparation-construction and legacy payload cleanup +remain separate required acceptance paths; they gain no authority from this path. + +A bounded `snapshot.reclamation.planned.1` event is committed under a deterministic +receipt-bound aggregate before private unlinks. It freezes each known stage leaf's +native identity, metadata generation, BLAKE3 content identity and size. The native +namespace lock, Location reader/mutation guard and exact exited lifetime remain +retained through deletion. Unknown entries, aliases or changed identities refuse; +no source image is read or installed by cleanup. Native Location/checkout metadata remains checked. Missing payloads are restart progress only +when matched to the already durable plan and original ownership metadata. + +The consuming Store writer transaction re-attests the original receipt, obtains a +complete bounded reference view, and refuses unsettled native owners in the same +Session/Location or an owner whose Location cannot be established. Reference +writers cannot interleave with the bounded native deletion callback. Independent +native activities and checkout records are never settled by cleanup. A separately +committed `snapshot.reclamation.completed.1` event retains cumulative reclaimed +bytes and the original plan identity for idempotent retry. The plan lives in the +Store, so a full snapshot quota cannot prevent recording cleanup eligibility. + +Retention uses the durable settlement time, with a fresh policy check before each +unlink. Disabled snapshots and zero retention preserve payloads. Cap pressure may +attempt this same eligible cleanup and retry capture once; it never overrides +reference, lifetime, acknowledgement or retention checks. Periodic fallback-object +collection also takes its reference view at the serialized Store writer, rather +than applying a stale view after later reference commits. Native intent/lifetime +metadata and all referenced CAS images remain retained by this initial path. + +The durable fallback reference view uses an event-owned index of canonical +`copy:<64 lowercase hex>` manifest identities. Migration backfills existing events +before the index can be used; transactional insert/update bookkeeping covers new +or rewritten event data and event deletion cascades remove only that event's roots. +The same complete index protects staged, inactive-branch and saved-version history. +A missing/unreadable/oversized index refuses cleanup. Reference lookup at the writer +must not repeatedly parse the entire conversation history while other writes wait. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md index 5c0af2f82..50e2ce232 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md @@ -28,3 +28,18 @@ - **WHEN** a database is upgraded for snapshot-worker ownership - **THEN** migration `20261106000000_snapshot_worker_ownership` persists without rewriting legacy evidence or adding tables - **AND** a binary whose migration catalog predates that marker opens the database read-only through the existing unknown-migration gate instead of writing while ignoring new native admission fences + +#### Scenario: Durable private snapshot reclamation plan +- **GIVEN** an ordinary file-copy restore has an exact committed settlement receipt and matching native acknowledgement +- **WHEN** eligible displaced payloads are reclaimed after retention +- **THEN** a bounded receipt-owned `snapshot.reclamation.planned.1` event SHALL commit before any private unlink +- **AND** the consuming writer SHALL revalidate the receipt and complete reference/uncertainty view while reference writes remain serialized +- **AND** `snapshot.reclamation.completed.1` SHALL retain the same plan identity and cumulative reclaimed bytes after durable filesystem completion +- **AND** an interrupted plan SHALL remain retryable without acknowledging independent native owners or replaying source effects + +#### Scenario: Transactional snapshot reference index +- **GIVEN** retained events contain canonical `copy:<64 lowercase hex>` manifest identities +- **WHEN** reference-index storage is introduced or event data is inserted, updated or deleted +- **THEN** existing references SHALL be backfilled and each later change SHALL update its event-owned reference rows in the same transaction +- **AND** shared references SHALL remain protected until every referencing event is removed +- **AND** missing, unreadable or oversized reference views SHALL refuse reclamation diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index a80ebd6c7..a373d29f8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -521,3 +521,14 @@ payloads and later source symlinks/bytes, messages and inference counts. These a local macOS arm64 positives; other platform/cut combinations remain unproven. M7/M8 and their parent tasks stay open alongside partial constructors, managed Git admission, safe reclamation and complete compatibility/platform evidence. + +M6 ordinary-payload continuation: implement receipt-bound Store plan/completion +records, writer-serialized reference checks, retained native namespace/reader/ +lifetime proof, and no-follow removal of positively owned ordinary file-copy +stages. Keep intent/lifetime metadata and CAS roots; recheck retention and refuse +unknown entries, aliases, live workers and independent uncertainty. Wire periodic +maintenance and a bounded cap-pressure capture retry. Validate real-App positives, +interrupted plan resume, missing/changed proof, disabled/zero retention and cap +pressure before accepting this path. Managed Git, recovery-resolution/preparation +construction payload cleanup and complete reclamation/platform acceptance remain +required; M6 and all parent tasks stay open. From f085c5ea3d3364710f093ee6e4871ae7f5fe3c10 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 11:24:09 -0400 Subject: [PATCH 43/87] Reclaim acknowledged recovery pair payloads --- Cargo.lock | 1 + crates/cyber-app/src/snapshot_reclamation.rs | 7 +- .../cyber-app/tests/snapshot_reclamation.rs | 31 +- crates/cyber-cli/Cargo.toml | 1 + crates/cyber-cli/tests/branches_cli.rs | 295 ++++++++++++ crates/cyber-cli/tests/support/reclamation.rs | 40 ++ .../src/runtime/snapshot_reclamation.rs | 163 +++++-- crates/cyber-snapshot/src/copy/lifetime.rs | 13 +- crates/cyber-snapshot/src/copy/reclamation.rs | 53 ++- .../src/copy/reclamation/paired.rs | 422 ++++++++++++++++++ crates/cyber-snapshot/src/reclamation.rs | 8 +- docs/rewind.md | 11 +- .../implementation.md | 75 ++++ .../managed-recovery.md | 22 + .../specs/storage-events/spec.md | 8 + .../complete-daily-coding-surfaces/tasks.md | 9 + 16 files changed, 1088 insertions(+), 71 deletions(-) create mode 100644 crates/cyber-cli/tests/support/reclamation.rs create mode 100644 crates/cyber-snapshot/src/copy/reclamation/paired.rs diff --git a/Cargo.lock b/Cargo.lock index 0f5cd9627..32eed6515 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1046,6 +1046,7 @@ dependencies = [ "cyber-llm", "cyber-sandbox", "cyber-server", + "cyber-snapshot", "cyber-store", "cyber-telemetry", "cyber-tools", diff --git a/crates/cyber-app/src/snapshot_reclamation.rs b/crates/cyber-app/src/snapshot_reclamation.rs index 588b459d1..dedf9ebd8 100644 --- a/crates/cyber-app/src/snapshot_reclamation.rs +++ b/crates/cyber-app/src/snapshot_reclamation.rs @@ -23,7 +23,7 @@ impl Policy for Reclamation { self.0 .read(move |db| { let tx = db.unchecked_transaction()?; - let Some(receipt) = records::restore_receipt(&tx, &origin)? else { + let Some(receipt) = records::settlement_receipt(&tx, &origin)? else { return Ok(None); }; let settled_at_ms = tx.query_row( @@ -39,10 +39,7 @@ impl Policy for Reclamation { }) .map_err(|error| error.to_string()) } - fn record( - &self, - receipt: &cyber_server::runtime::SnapshotRestoreReceipt, - ) -> Result, String> { + fn record(&self, receipt: &records::Receipt) -> Result, String> { let receipt = receipt.clone(); self.0 .read(move |db| { diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs index 9f7949d0f..5c3179d3e 100644 --- a/crates/cyber-app/tests/snapshot_reclamation.rs +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -227,7 +227,15 @@ async fn prompt(app: &App, client: &Client, session: &str, text: &str, expected: #[tokio::test] async fn acknowledged_ordinary_payloads_are_reclaimed_without_source_or_reference_image_changes() { + use sha2::Digest; let f = Fixture::new().await; + let legacy_frame = serde_json::to_vec(&("snapshot-reclamation-v1", &f.receipt)).unwrap(); + let legacy_id = format!("snapshot_reclaim_{:x}", sha2::Sha256::digest(legacy_frame)); + assert_eq!(records::Plan::id(&f.receipt).unwrap(), legacy_id); + assert_eq!( + serde_json::to_value(records::Receipt::Restore(f.receipt.clone())).unwrap(), + serde_json::to_value(&f.receipt).unwrap() + ); f.age(); let messages = f .client @@ -364,7 +372,7 @@ impl Policy for Cut { ) -> Result, String> { self.inner.receipt(origin) } - fn record(&self, receipt: &SnapshotRestoreReceipt) -> Result, String> { + fn record(&self, receipt: &records::Receipt) -> Result, String> { self.inner.record(receipt) } fn reclaim(&self, plan: records::Plan, delete: Delete) -> Result { @@ -507,14 +515,26 @@ async fn actual_process_death_preserves_plan_and_resumes_without_source_replay() .unwrap(), ); let policy = cyber_app::snapshot_reclamation::policy(store.clone()); - assert!(!policy.record(&receipt).unwrap().unwrap().completed); + assert!( + !policy + .record(&receipt.clone().into()) + .unwrap() + .unwrap() + .completed + ); let snapshots = maintenance( &paths, Arc::new(Mutex::new(json!({"snapshots":{"retention_days":7}}))), policy.clone(), ); assert!(snapshots.reclaim(project.to_str().unwrap()).await.unwrap() > 0); - assert!(policy.record(&receipt).unwrap().unwrap().completed); + assert!( + policy + .record(&receipt.clone().into()) + .unwrap() + .unwrap() + .completed + ); assert_eq!( snapshots.reclaim(project.to_str().unwrap()).await.unwrap(), 0 @@ -775,7 +795,10 @@ async fn reclamation_projectors_roll_back_foreign_plans_and_invalid_completions( let id = records::Plan::id(&plan.receipt).unwrap(); let notifications = f.app.store.subscribe(); let mut wrong = plan.clone(); - wrong.receipt.events_digest = "0".repeat(64); + let records::Receipt::Restore(receipt) = &mut wrong.receipt else { + panic!("ordinary fixture"); + }; + receipt.events_digest = "0".repeat(64); let foreign = records::Plan::id(&wrong.receipt).unwrap(); assert!( f.app diff --git a/crates/cyber-cli/Cargo.toml b/crates/cyber-cli/Cargo.toml index 2eedd1f37..05b4c95f3 100644 --- a/crates/cyber-cli/Cargo.toml +++ b/crates/cyber-cli/Cargo.toml @@ -57,6 +57,7 @@ tempfile.workspace = true chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } [dev-dependencies] +cyber-snapshot = { path = "../cyber-snapshot" } cyber-install = { path = "../cyber-install", features = ["test-support"] } zip = { version = "8.6", default-features = false, features = ["deflate-flate2"] } aws-lc-rs = "1" diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 78932bccf..194bdf1b8 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -10,6 +10,9 @@ use std::time::Duration; use serde_json::{Value, json}; +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[path = "support/reclamation.rs"] +mod reclamation_support; #[path = "../../cyber-app/tests/support/scripted_model.rs"] mod scripted_model; @@ -1253,6 +1256,7 @@ fn managed_native_recovery_after_partial_kill(action: &str) { std::fs::write(repo.join(format!("bulk_{index:04}")), "after").unwrap(); } std::fs::write(repo.join("zz\nfixture"), "after").unwrap(); + std::fs::write(repo.join("aa_removed"), "after-only source").unwrap(); env.exec(Some(&id), "after snapshot"); let messages = kill_partial_restore(&env, &id, &first); let requests = model.requests().len(); @@ -1288,9 +1292,300 @@ fn managed_native_recovery_after_partial_kill(action: &str) { ); assert_native_settlement_link(&env, &journal, &result); assert_recovery_lookup_does_not_replay(&env, &id, &request_id, &request, &result, &messages); + assert_cleanup_preserves_request(&env, &id, &request_id, &result, &messages); assert_eq!(model.requests().len(), requests); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_cleanup_preserves_request( + env: &Env, + id: &str, + request_id: &str, + result: &Value, + messages: &[Value], +) { + assert_paired_payload_cleanup(env, result); + let lookup = json(&env.cyber(&[ + "sessions", + "recovery-request", + id, + request_id, + "--acknowledge", + "--format", + "json", + ])); + assert_eq!(lookup["receipt"], result["receipt"]); + assert_eq!(env.messages(id), messages); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_paired_payload_cleanup(env: &Env, result: &Value) { + use cyber_server::runtime::snapshot_reclamation::{self as records, Receipt}; + use cyber_store::{Store, StoreOptions}; + use std::sync::Arc; + exits(&env.cyber(&["service", "stop"]), 0); + let root = env.root.join("cyber-home"); + let data = root.join("data"); + let database = std::fs::read_dir(&data) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|extension| extension == "db")) + .unwrap(); + let store = Arc::new( + Store::open(StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(database), + cyber_server::runtime::Runtime::registry(), + )) + .unwrap(), + ); + let receipt: cyber_server::runtime::SnapshotRecoveryReceipt = + serde_json::from_value(result["receipt"].clone()).unwrap(); + let owner = receipt + .checkout + .as_ref() + .unwrap() + .resolution_origin + .activity_id + .clone(); + let receipt = Receipt::Recovery(receipt); + let namespace = std::fs::read_dir(data.join("snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let journals = match &receipt { + Receipt::Recovery(receipt) => [ + namespace.join("journals").join(&receipt.journal_id), + namespace.join("journals").join(&receipt.resolution_id), + ], + _ => unreachable!(), + }; + let intents = journals + .each_ref() + .map(|journal| std::fs::read(journal.join("intent.json")).unwrap()); + let original_source = env.file("bulk_0000"); + let source = env.root.join("repo"); + let target = env.root.join("cleanup-owner-target"); + std::fs::write(&target, "later owner target").unwrap(); + std::os::unix::fs::symlink(&target, source.join("cleanup-owner-link")).unwrap(); + let policy = cyber_app::snapshot_reclamation::policy(store.clone()); + let snapshots = cyber_snapshot::GitSnapshots::with_reclamation( + data.clone(), + Arc::new(|_| json!({"snapshots":{"retention_days":7}})), + vec![root.join("config"), root.join("state"), root.join("cache")], + None, + Some(policy.clone()), + ); + let runtime = tokio::runtime::Runtime::new().unwrap(); + assert_eq!( + runtime + .block_on(snapshots.reclaim(source.to_str().unwrap())) + .unwrap(), + 0, + "young paired receipt must retain payloads" + ); + assert!(policy.record(&receipt).unwrap().is_none()); + store + .transaction(move |db| { + db.execute( + "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", + rusqlite::params![8i64 * 86400000, owner], + )?; + Ok(()) + }) + .unwrap(); + assert_paired_live_owners_refuse(&runtime, &snapshots, &namespace, &receipt); + let injection = Arc::new(std::sync::Mutex::new(None)); + let cut = cyber_snapshot::GitSnapshots::with_reclamation( + data, + Arc::new(|_| json!({"snapshots":{"retention_days":7}})), + vec![root.join("config"), root.join("state"), root.join("cache")], + None, + Some(Arc::new(reclamation_support::Planned( + policy.clone(), + injection.clone(), + ))), + ); + assert_paired_plan_interruption( + &runtime, + &snapshots, + (&cut, &injection), + &policy, + &receipt, + &namespace, + &journals, + ); + assert!( + runtime + .block_on(snapshots.reclaim(source.to_str().unwrap())) + .unwrap() + > 0 + ); + let record = policy.record(&receipt).unwrap().unwrap(); + assert!(record.completed); + assert_eq!(record.plan.version, 2); + assert_eq!( + records::Plan::id(&record.plan.receipt).unwrap(), + records::Plan::id(&receipt).unwrap() + ); + for (journal, intent) in journals.iter().zip(intents) { + assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); + assert_eq!( + std::fs::read_dir(journal).unwrap().count(), + 1, + "only the acknowledged intent remains" + ); + } + assert_eq!( + runtime + .block_on(snapshots.reclaim(source.to_str().unwrap())) + .unwrap(), + 0 + ); + assert_eq!(env.file("bulk_0000"), original_source); + assert_eq!( + std::fs::read_link(source.join("cleanup-owner-link")).unwrap(), + target + ); + assert_eq!( + std::fs::read_to_string(target).unwrap(), + "later owner target" + ); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_paired_plan_interruption( + runtime: &tokio::runtime::Runtime, + snapshots: &cyber_snapshot::GitSnapshots, + cut: ( + &cyber_snapshot::GitSnapshots, + &std::sync::Arc>>, + ), + policy: &std::sync::Arc, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, + namespace: &std::path::Path, + journals: &[PathBuf; 2], +) { + assert!( + runtime + .block_on( + cut.0.reclaim( + receipt + .origin() + .unwrap() + .binding + .directory + .to_str() + .unwrap() + ) + ) + .is_err() + ); + let pending = policy.record(receipt).unwrap().unwrap(); + assert!(!pending.completed); + let first = &pending.plan.files[0]; + let leaf = namespace + .join("journals") + .join(first.journal.as_deref().unwrap()) + .join(&first.name); + let unknown = journals[1].join("foreign-entry"); + std::fs::write(&unknown, "owner evidence").unwrap(); + assert!( + runtime + .block_on( + snapshots.reclaim( + receipt + .origin() + .unwrap() + .binding + .directory + .to_str() + .unwrap() + ) + ) + .is_err() + ); + assert!( + leaf.exists(), + "unknown entry must refuse before any planned unlink" + ); + std::fs::remove_file(unknown).unwrap(); + assert_unplanned_empty_slot(runtime, cut, &journals[1], receipt, &leaf); + // Synthetic state cut of one planned unlink; the restore itself was killed + // as a real process above. This is not a paired cleanup process-kill claim. + std::fs::remove_file(&leaf).unwrap(); + std::fs::File::open(leaf.parent().unwrap()) + .unwrap() + .sync_all() + .unwrap(); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_unplanned_empty_slot( + runtime: &tokio::runtime::Runtime, + cut: ( + &cyber_snapshot::GitSnapshots, + &std::sync::Arc>>, + ), + journal: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, + retained: &std::path::Path, +) { + use cyber_server::runtime::{SnapshotRecoveryAction, snapshot_reclamation::Receipt}; + let Receipt::Recovery(pair) = receipt else { + panic!("paired fixture"); + }; + if pair.action != SnapshotRecoveryAction::Rollback { + return; + } + let intent: Value = + serde_json::from_slice(&std::fs::read(journal.join("intent.json")).unwrap()).unwrap(); + let index = intent["changes"] + .as_array() + .unwrap() + .iter() + .position(|change| change["before"].is_null()) + .expect("rollback fixture needs a genuinely empty private stage"); + let path = journal.join(format!("stage_{index}")); + assert!(!path.exists()); + *cut.1.lock().unwrap() = Some(path.clone()); + let error = runtime + .block_on( + cut.0 + .reclaim(pair.original.binding.directory.to_str().unwrap()), + ) + .unwrap_err(); + assert!(error.contains("Unplanned entry"), "{error}"); + assert!( + retained.exists(), + "no planned payload may be removed before complete preflight" + ); + assert_eq!(std::fs::read(&path).unwrap(), b"unplanned owner evidence"); + std::fs::remove_file(path).unwrap(); + *cut.1.lock().unwrap() = None; +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_paired_live_owners_refuse( + runtime: &tokio::runtime::Runtime, + snapshots: &cyber_snapshot::GitSnapshots, + namespace: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + for (origin, _) in receipt.owners().unwrap() { + let held = + std::fs::File::open(namespace.join("workers").join(&origin.activity_id)).unwrap(); + held.try_lock().unwrap(); + assert!( + runtime + .block_on(snapshots.reclaim(origin.binding.directory.to_str().unwrap())) + .is_err() + ); + drop(held); + } +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn kill_partial_restore(env: &Env, id: &str, first: &str) -> Vec { let selection = reviewed_selection(env, first, "both", "commit"); diff --git a/crates/cyber-cli/tests/support/reclamation.rs b/crates/cyber-cli/tests/support/reclamation.rs new file mode 100644 index 000000000..d2a5e4978 --- /dev/null +++ b/crates/cyber-cli/tests/support/reclamation.rs @@ -0,0 +1,40 @@ +//! Stop after a durable plan; native payload deletion is deliberately not run. +use cyber_server::runtime::{ + SnapshotOrigin, + snapshot_reclamation::{Plan, Receipt, Record}, +}; +use cyber_snapshot::reclamation::{Collect, Delete, Policy, Settlement}; +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, +}; + +pub struct Planned(pub Arc, pub Arc>>); +impl Policy for Planned { + fn receipt(&self, origin: &SnapshotOrigin) -> Result, String> { + self.0.receipt(origin) + } + fn record(&self, receipt: &Receipt) -> Result, String> { + self.0.record(receipt) + } + fn reclaim(&self, plan: Plan, delete: Delete) -> Result { + let injection = self.1.lock().unwrap().clone(); + self.0.reclaim( + plan, + Box::new(move || { + if let Some(path) = injection { + std::fs::write(path, b"unplanned owner evidence").unwrap(); + return delete(); + } + let _native_guards = delete; + Err("fixture cut after durable paired plan".into()) + }), + ) + } + fn collect(&self, collect: Collect) -> Result<(), String> { + self.0.collect(collect) + } + fn refused(&self, directory: &str, error: &str) { + self.0.refused(directory, error); + } +} diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation.rs b/crates/cyber-server/src/runtime/snapshot_reclamation.rs index ae80ee9a3..d32626ecb 100644 --- a/crates/cyber-server/src/runtime/snapshot_reclamation.rs +++ b/crates/cyber-server/src/runtime/snapshot_reclamation.rs @@ -1,5 +1,8 @@ //! Durable private-payload cleanup plans. These never settle native activities. -use super::{SnapshotOrigin, SnapshotRestoreReceipt, activity}; +use super::{ + SnapshotOrigin, SnapshotRecoveryReceipt, SnapshotRestoreCheckout, SnapshotRestoreReceipt, + activity, +}; use cyber_store::{EventRegistry, StoreError, StoredEvent}; use rusqlite::{Connection, OptionalExtension}; use serde::{Deserialize, Serialize}; @@ -11,9 +14,80 @@ pub const COMPLETED: &str = "snapshot.reclamation.completed.1"; const LIMIT: usize = 4096; const BYTES: usize = 8 * 1024 * 1024; +/// Untagged serialization preserves every existing ordinary receipt and plan ID. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Receipt { + Restore(SnapshotRestoreReceipt), + Recovery(SnapshotRecoveryReceipt), +} +impl From for Receipt { + fn from(value: SnapshotRestoreReceipt) -> Self { + Self::Restore(value) + } +} +impl From for Receipt { + fn from(value: SnapshotRecoveryReceipt) -> Self { + Self::Recovery(value) + } +} +impl Receipt { + pub fn origin(&self) -> Result<&SnapshotOrigin, StoreError> { + match self { + Self::Restore(receipt) => Ok(&receipt.origin), + Self::Recovery(receipt) => receipt + .checkout + .as_ref() + .map(|checkout| &checkout.resolution_origin) + .ok_or_else(|| invalid("Missing recovery checkout")), + } + } + pub fn owners(&self) -> Result, StoreError> { + match self { + Self::Restore(receipt) => { + receipt.checkout_acknowledgement_digest().map_err(invalid)?; + Ok(vec![( + &receipt.origin, + receipt + .checkout + .as_deref() + .ok_or_else(|| invalid("Missing restore checkout"))?, + )]) + } + Self::Recovery(receipt) => { + receipt.checkout_acknowledgement_digest().map_err(invalid)?; + let checkout = receipt + .checkout + .as_ref() + .ok_or_else(|| invalid("Missing recovery checkout"))?; + Ok(vec![ + ( + &receipt.original, + checkout + .original + .as_deref() + .ok_or_else(|| invalid("Missing original recovery checkout"))?, + ), + (&checkout.resolution_origin, &checkout.resolution), + ]) + } + } + } + fn verify(&self, db: &Connection) -> Result<(), StoreError> { + match self { + Self::Restore(receipt) => super::snapshot_restore_receipt::verify(db, receipt), + Self::Recovery(receipt) => { + super::snapshot_acknowledgement::verify_committed(db, receipt) + } + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct File { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub journal: Option, pub name: String, pub identity: String, pub generation: String, @@ -24,7 +98,7 @@ pub struct File { #[serde(deny_unknown_fields)] pub struct Plan { pub version: u32, - pub receipt: SnapshotRestoreReceipt, + pub receipt: Receipt, pub files: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -45,28 +119,33 @@ fn hash(value: &str) -> bool { value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) } impl Plan { - pub fn id(receipt: &SnapshotRestoreReceipt) -> Result { + pub fn id>(receipt: &R) -> Result { + let receipt: Receipt = receipt.clone().into(); let frame = serde_json::to_vec(&("snapshot-reclamation-v1", receipt)) .map_err(|e| invalid(e.to_string()))?; Ok(format!("snapshot_reclaim_{:x}", Sha256::digest(frame))) } pub fn validate(&self) -> Result<(), StoreError> { - let checkout = self - .receipt - .checkout - .as_ref() - .ok_or_else(|| invalid("Reclamation requires retained checkout evidence"))?; - self.receipt - .checkout_acknowledgement_digest() - .map_err(invalid)?; - let empty = checkout - .lifetime - .checkout_resources_digest(&[]) - .map_err(invalid)?; - if self.version != 1 - || self.receipt.origin.worktree_id.is_some() - || checkout.resources_digest != empty - || self.files.len() > LIMIT + let owners = self.receipt.owners()?; + for (origin, checkout) in &owners { + if origin.worktree_id.is_some() + || checkout.resources_digest + != checkout + .lifetime + .checkout_resources_digest(&[]) + .map_err(invalid)? + { + return Err(invalid( + "Managed reclamation requires retained resource guards", + )); + } + } + let version = match &self.receipt { + Receipt::Restore(_) => 1, + Receipt::Recovery(_) => 2, + }; + if self.version != version + || self.files.len() > LIMIT * owners.len() || serde_json::to_vec(self) .map_err(|e| invalid(e.to_string()))? .len() @@ -83,7 +162,13 @@ impl Plan { .strip_prefix("stage_") .and_then(|index| index.parse::().ok()); if index.is_none_or(|index| index >= LIMIT || file.name != format!("stage_{index}")) - || !names.insert(&file.name) + || !names.insert((&file.journal, &file.name)) + || match &self.receipt { + Receipt::Restore(_) => file.journal.is_some(), + Receipt::Recovery(_) => !owners.iter().any(|(origin, _)| { + file.journal.as_deref() == Some(origin.binding.journal_id.as_str()) + }), + } || file.identity.is_empty() || file.identity.len() > 128 || !hash(&file.generation) @@ -105,18 +190,17 @@ impl Plan { /// must separately retain native namespace, reader, lifetime and lease proof. pub fn authorize(&self, db: &Connection) -> Result, StoreError> { self.validate()?; - super::snapshot_restore_receipt::verify(db, &self.receipt)?; - let journal = &self.receipt.origin.binding.journal_id; - let directory = self - .receipt - .origin + self.receipt.verify(db)?; + let origin = self.receipt.origin()?; + let journal = &origin.binding.journal_id; + let directory = origin .binding .directory .to_str() .ok_or_else(|| invalid("Non-UTF8 reclamation Location"))?; let protected: bool = db.query_row( - "SELECT EXISTS(SELECT 1 FROM event e LEFT JOIN session s ON s.id=json_extract(e.data,'$.session_id') WHERE e.type='native.activity.changed.1' AND coalesce(json_extract(e.data,'$.status'),'unknown')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) AND (s.id IS NULL OR (coalesce(json_extract(e.data,'$.status'),'unknown')='unknown' AND json_extract(e.data,'$.snapshot_lifetime') IS NULL) OR s.directory=?2 OR json_extract(e.data,'$.session_id')=?3 OR json_extract(e.data,'$.snapshot_lifetime.owner.directory')=?2 OR EXISTS(SELECT 1 FROM json_tree(e.data) j WHERE j.key='journal_id' AND j.value=?1)))", - rusqlite::params![journal, directory, self.receipt.origin.session_id], |row| row.get(0), + "SELECT EXISTS(SELECT 1 FROM event e LEFT JOIN session s ON s.id=json_extract(e.data,'$.session_id') WHERE e.type='native.activity.changed.1' AND coalesce(json_extract(e.data,'$.status'),'unknown')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) AND (s.id IS NULL OR (coalesce(json_extract(e.data,'$.status'),'unknown')='unknown' AND json_extract(e.data,'$.snapshot_lifetime') IS NULL) OR s.directory=?2 OR json_extract(e.data,'$.session_id')=?3 OR json_extract(e.data,'$.snapshot_lifetime.owner.directory')=?2 OR EXISTS(SELECT 1 FROM json_tree(e.data) j WHERE j.key='journal_id' AND j.value IN (?1,?4))))", + rusqlite::params![journal, directory, origin.session_id, self.receipt.owners()?[0].0.binding.journal_id], |row| row.get(0), )?; if protected { return Err(invalid( @@ -142,10 +226,10 @@ pub fn references(db: &Connection) -> Result, StoreError> { Ok(roots) } -pub fn restore_receipt( +pub fn settlement_receipt( db: &Connection, origin: &SnapshotOrigin, -) -> Result, StoreError> { +) -> Result, StoreError> { let raw: Option = db .query_row( "SELECT data FROM event WHERE aggregate_id=?1 AND type=?2 ORDER BY seq DESC LIMIT 1", @@ -158,21 +242,26 @@ pub fn restore_receipt( }; let record: activity::Record = serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; - let Some(receipt) = record.snapshot_restore_receipt else { - return Ok(None); + let receipt = match (record.snapshot_restore_receipt, record.snapshot_receipt) { + (Some(receipt), None) => Receipt::Restore(receipt), + (None, Some(receipt)) => Receipt::Recovery(receipt), + (None, None) => return Ok(None), + _ => return Err(invalid("Ambiguous cleanup receipt")), }; - if receipt.origin != *origin { + receipt.owners()?; + if receipt.origin()? != origin { return Err(invalid("Foreign reclamation receipt")); } - super::snapshot_restore_receipt::verify(db, &receipt)?; + receipt.verify(db)?; Ok(Some(receipt)) } -pub fn lookup( +pub fn lookup>( db: &Connection, - receipt: &SnapshotRestoreReceipt, + receipt: &R, ) -> Result, StoreError> { - let id = Plan::id(receipt)?; + let receipt: Receipt = receipt.clone().into(); + let id = Plan::id(&receipt)?; let first: Option<(String, String)> = db .query_row( "SELECT type,data FROM event WHERE aggregate_id=?1 AND seq=0", @@ -185,7 +274,7 @@ pub fn lookup( }; let plan: Plan = serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; plan.validate()?; - if kind != PLANNED || plan.receipt != *receipt { + if kind != PLANNED || plan.receipt != receipt { return Err(invalid("Reclamation request owner differs")); } let (seq, kind, raw): (i64, String, String) = db.query_row( diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index 6ea97f97d..191963c24 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -223,14 +223,23 @@ impl Store { binding: &SnapshotLifetimeBinding, origin: &cyber_server::runtime::SnapshotOrigin, resources_digest: &str, + ) -> Result<(), String> { + self.verify_reclamation_phase_admission(binding, origin, resources_digest, true) + } + pub(crate) fn verify_reclamation_phase_admission( + &self, + binding: &SnapshotLifetimeBinding, + origin: &cyber_server::runtime::SnapshotOrigin, + resources_digest: &str, + admitted: bool, ) -> Result<(), String> { let phase = self.read_source_phase(binding)?; let resources = phase .checkout_bindings .as_ref() .ok_or("Missing acquired reclamation resources")?; - if !phase.admitted - || phase.origin.as_ref() != Some(origin) + if phase.admitted != admitted + || phase.origin.as_ref() != admitted.then_some(origin) || !resources.is_empty() || binding.checkout_resources_digest(resources)? != resources_digest { diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index 150e5305e..3528ff820 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -1,4 +1,5 @@ //! Reclaim only exact acknowledged displaced images; keep intent/lifetime receipts. +mod paired; use super::{ Store, ownership, restore::{Journal, JournalBackend}, @@ -28,6 +29,7 @@ struct Prepared { workspace: tokio::sync::OwnedMutexGuard<()>, config: Arc, settled_at_ms: i64, + intent_digest: String, } struct MetadataPin { file: std::fs::File, @@ -128,7 +130,7 @@ pub(crate) fn locations(data: &Path) -> Result, String> { )?) .map_err(storage::fail)?, }; - if ordinary(&record) { + if ordinary(&record) || paired::eligible(&record) { result.insert( record .location @@ -153,6 +155,9 @@ pub(crate) fn reclaim( return Ok(0); } let (directory, record) = store.journal(id)?; + if paired::eligible(&record) { + return paired::reclaim(store, workspace, config, policy, directory, record); + } if !ordinary(&record) { return Ok(0); } @@ -193,12 +198,15 @@ fn prepare( config: Arc, ) -> Result { let receipt = settlement.receipt; - let origin = &receipt.origin; + let cyber_server::runtime::snapshot_reclamation::Receipt::Restore(restore) = &receipt else { + return Err("Ordinary cleanup received a recovery receipt".into()); + }; + let origin = &restore.origin; store.verify_worker_namespace(origin)?; - if store.worker_digest(Some(origin))?.as_ref() != Some(&receipt.intent_digest) { + if store.worker_digest(Some(origin))?.as_ref() != Some(&restore.intent_digest) { return Err("Reclamation intent differs from committed restore".into()); } - let checkout = receipt + let checkout = restore .checkout .as_ref() .ok_or("Legacy restore has no cleanup checkout proof")?; @@ -239,6 +247,7 @@ fn prepare( let pin = storage::open(&journal, Path::new(&name))?; ownership.verify_original(index, &pin)?; let file = Payload { + journal: None, name: name.clone(), identity: image.identity, generation: image.generation, @@ -268,6 +277,7 @@ fn prepare( return Err("Unknown journal file or construction stage; reclamation refused".into()); } } + let intent_digest = restore.intent_digest.clone(); let plan = Plan { version: 1, receipt, @@ -287,18 +297,20 @@ fn prepare( let phase = MetadataPin::open(&workers, name)?; store.verify_reclamation_phase( binding, - &plan.receipt.origin, - &plan - .receipt - .checkout - .as_ref() - .ok_or("Missing reclamation checkout")? + plan.receipt.origin().map_err(storage::fail)?, + &plan.receipt.owners().map_err(storage::fail)?[0] + .1 .resources_digest, )?; phase.verify(&workers, name)?; storage::existing_child( &store.journals, - &plan.receipt.origin.binding.journal_id, + &plan + .receipt + .origin() + .map_err(storage::fail)? + .binding + .journal_id, true, )?; Ok(Prepared { @@ -312,6 +324,7 @@ fn prepare( workspace, config, settled_at_ms: settlement.settled_at_ms, + intent_digest, }) } fn unmanaged(store: &Store) -> Result<(), String> { @@ -328,7 +341,7 @@ impl Prepared { let _reader_lease = &self.workspace; self.lifetime.verify()?; self.store - .verify_worker_namespace(&self.plan.receipt.origin)?; + .verify_worker_namespace(self.plan.receipt.origin().map_err(storage::fail)?)?; if !due(self.config.as_ref(), &self.store.root, self.settled_at_ms) { return Err("Snapshot retention no longer permits reclamation".into()); } @@ -345,15 +358,21 @@ impl Prepared { } fn verify(&self) -> Result<(), String> { self.verify_routes()?; - let (_, record) = self - .store - .journal(&self.plan.receipt.origin.binding.journal_id)?; + let (_, record) = self.store.journal( + &self + .plan + .receipt + .origin() + .map_err(storage::fail)? + .binding + .journal_id, + )?; if !ordinary(&record) || self .store - .worker_digest(Some(&self.plan.receipt.origin))? + .worker_digest(Some(self.plan.receipt.origin().map_err(storage::fail)?))? .as_ref() - != Some(&self.plan.receipt.intent_digest) + != Some(&self.intent_digest) { return Err("Acknowledged reclamation intent changed".into()); } diff --git a/crates/cyber-snapshot/src/copy/reclamation/paired.rs b/crates/cyber-snapshot/src/copy/reclamation/paired.rs new file mode 100644 index 000000000..4a0ab5d0a --- /dev/null +++ b/crates/cyber-snapshot/src/copy/reclamation/paired.rs @@ -0,0 +1,422 @@ +//! A committed recovery pair owns two private inventories, never current source. +use super::*; +use cyber_server::runtime::{ + SnapshotOrigin, SnapshotRestoreCheckout, snapshot_reclamation::Receipt, +}; + +struct Owner { + origin: SnapshotOrigin, + journal: Dir, + intent: MetadataPin, + digest: String, + phase: MetadataPin, + lifetime: super::super::lifetime::Lifetime, +} +struct PreparedPair { + store: Store, + owners: Vec, + workers: Dir, + plan: Plan, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + settled_at_ms: i64, +} +pub(super) fn eligible(record: &Journal) -> bool { + record.completed + && record.acknowledged + && record.resolution.is_some() + && record.settled_by.is_none() + && record.origin.is_some() + && record + .context + .as_ref() + .is_some_and(|context| context.git.is_none()) + && record.backend != Some(JournalBackend::Git) +} +pub(super) fn reclaim( + store: Store, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + policy: Arc, + journal: Dir, + record: Journal, +) -> Result { + let origin = record.origin.as_ref().ok_or("Missing recovery origin")?; + let Some(settlement) = policy.receipt(origin)? else { + return Ok(0); + }; + if !due(config.as_ref(), &store.root, settlement.settled_at_ms) { + return Ok(0); + } + let existing = policy.record(&settlement.receipt)?; + if existing.as_ref().is_some_and(|record| record.completed) { + return Ok(0); + } + let prepared = prepare( + store, + workspace, + config, + journal, + record, + settlement, + existing.map(|record| record.plan), + )?; + if prepared.plan.files.is_empty() { + return Ok(0); + } + policy.reclaim(prepared.plan.clone(), Box::new(move || prepared.delete())) +} + +fn owner( + store: &Store, + workers: &Dir, + origin: &SnapshotOrigin, + checkout: &SnapshotRestoreCheckout, + journal: Dir, + digest: String, + admitted: bool, +) -> Result { + store.verify_worker_namespace(origin)?; + if store.worker_digest(Some(origin))?.as_ref() != Some(&digest) { + return Err("Recovery cleanup intent differs".into()); + } + let lifetime = store.lock_lifetime(&checkout.lifetime)?; + let resources = lifetime + .checkout_bindings()? + .ok_or("Missing recovery checkout inventory")?; + if !resources.is_empty() + || checkout.resources_digest != checkout.lifetime.checkout_resources_digest(&resources)? + { + return Err("Managed recovery cleanup requires retained resource guards".into()); + } + let name = checkout + .lifetime + .source_phase + .as_deref() + .ok_or("Missing recovery phase")?; + let phase = MetadataPin::open(workers, name)?; + store.verify_reclamation_phase_admission( + &checkout.lifetime, + origin, + &checkout.resources_digest, + admitted, + )?; + phase.verify(workers, name)?; + let intent = MetadataPin::open(&journal, "intent.json")?; + Ok(Owner { + origin: origin.clone(), + journal, + intent, + digest, + phase, + lifetime, + }) +} + +fn prepare( + store: Store, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + journal: Dir, + record: Journal, + settlement: Settlement, + frozen: Option, +) -> Result { + let Receipt::Recovery(receipt) = &settlement.receipt else { + return Err("Recovery cleanup requires its paired receipt".into()); + }; + receipt.checkout_acknowledgement_digest()?; + let checkout = receipt + .checkout + .as_ref() + .ok_or("Missing paired checkout proof")?; + let original_checkout = checkout + .original + .as_deref() + .ok_or("Missing original checkout proof")?; + let resolution = record + .resolution + .as_ref() + .ok_or("Missing resolution intent")?; + let (parent_dir, parent) = store.journal(&receipt.journal_id)?; + store.verify_settlement(&parent)?; + store.verify_receipt(record.origin.as_ref(), receipt)?; + if !eligible(&record) + || parent.origin.as_ref() != Some(&receipt.original) + || parent.settled_by.as_deref() != Some(receipt.resolution_id.as_str()) + || parent + .context + .as_ref() + .is_none_or(|context| context.git.is_some()) + { + return Err("Recovery cleanup pair or backend changed".into()); + } + unmanaged(&store)?; + let workers = storage::existing_child(&store.namespace, "workers", true)?; + let original = owner( + &store, + &workers, + &receipt.original, + original_checkout, + parent_dir, + resolution.original_digest.clone(), + true, + )?; + let resolved = owner( + &store, + &workers, + &checkout.resolution_origin, + &checkout.resolution, + journal, + receipt + .resolution_digest + .clone() + .ok_or("Missing resolution digest")?, + !record.changes.is_empty(), + )?; + let mut files = original_payloads(&store, &original, &parent, resolution, frozen.as_ref())?; + files.extend(resolution_payloads( + &store, + &resolved, + &record, + frozen.as_ref(), + )?); + let plan = Plan { + version: 2, + receipt: settlement.receipt, + files, + }; + plan.validate().map_err(storage::fail)?; + if frozen.as_ref().is_some_and(|frozen| frozen != &plan) { + return Err("Recovery cleanup plan differs from retained request".into()); + } + let prepared = PreparedPair { + store, + owners: vec![original, resolved], + workers, + plan, + workspace, + config, + settled_at_ms: settlement.settled_at_ms, + }; + prepared.verify()?; + Ok(prepared) +} + +fn inventory(journal: &Dir, count: usize) -> Result<(), String> { + let known = (0..count) + .map(|index| format!("stage_{index}")) + .chain(std::iter::once("intent.json".into())) + .collect::>(); + if storage::names(journal, super::super::ENTRY_LIMIT * 3)? + .iter() + .any(|name| !known.contains(name)) + { + return Err("Unknown recovery payload or construction entry".into()); + } + Ok(()) +} +fn planned<'a>(frozen: Option<&'a Plan>, owner: &Owner) -> BTreeMap<&'a str, &'a Payload> { + frozen + .into_iter() + .flat_map(|plan| &plan.files) + .filter(|file| file.journal.as_deref() == Some(owner.origin.binding.journal_id.as_str())) + .map(|file| (file.name.as_str(), file)) + .collect() +} + +fn payload(owner: &Owner, name: String, image: ownership::ImageObservation) -> Payload { + Payload { + journal: Some(owner.origin.binding.journal_id.clone()), + name, + identity: image.identity, + generation: image.generation, + hash: image.entry.object, + bytes: image.entry.length, + } +} +fn original_payloads( + store: &Store, + owner: &Owner, + parent: &Journal, + resolution: &super::super::resolution::Resolution, + frozen: Option<&Plan>, +) -> Result, String> { + let outcome = resolution + .outcome + .as_ref() + .ok_or("Missing immutable recovery outcome")?; + if outcome.files.len() != parent.changes.len() { + return Err("Incomplete recovery slot inventory".into()); + } + inventory(&owner.journal, parent.changes.len())?; + let mut files = Vec::new(); + let planned = planned(frozen, owner); + for (index, slots) in outcome.files.iter().enumerate() { + let name = format!("stage_{index}"); + let actual = + ownership::observe_image(store, Some(&owner.journal), std::ffi::OsStr::new(&name))?; + let expected = slots.stage.as_ref(); + let file = expected + .cloned() + .map(|image| payload(owner, name.clone(), image)); + if expected.is_some_and(|image| image.entry.kind != super::super::ImageKind::File) + || (actual.as_ref() != expected + && !(actual.is_none() && file.as_ref() == planned.get(name.as_str()).copied())) + { + return Err("Original recovery private slot differs from acknowledged outcome".into()); + } + if let Some(file) = file { + files.push(file); + } + } + Ok(files) +} +fn resolution_payloads( + store: &Store, + owner: &Owner, + record: &Journal, + frozen: Option<&Plan>, +) -> Result, String> { + inventory(&owner.journal, record.changes.len())?; + let ownership = record + .ownership + .as_ref() + .ok_or("Missing resolution payload ownership")?; + let mut files = Vec::new(); + let planned = planned(frozen, owner); + for (index, change) in record.changes.iter().enumerate() { + let name = format!("stage_{index}"); + let actual = + ownership::observe_image(store, Some(&owner.journal), std::ffi::OsStr::new(&name))?; + let previous = planned.get(name.as_str()).copied(); + match (&change.before, actual) { + (Some(before), Some(image)) + if before.kind == super::super::ImageKind::File && image.entry == *before => + { + let pin = storage::open(&owner.journal, Path::new(&name))?; + ownership.verify_original(index, &pin)?; + files.push(payload(owner, name, image)); + } + (Some(before), None) + if previous.is_some_and(|file| { + file.hash == before.object + && file.bytes == before.length + && ownership.original_identity(index) == Some(file.identity.as_str()) + }) => + { + files.push( + previous + .cloned() + .ok_or("Missing planned resolution payload")?, + ) + } + (None, None) if previous.is_none() => {} + _ => return Err("Resolution displaced payload lacks original ownership".into()), + } + } + Ok(files) +} +impl PreparedPair { + fn routes(&self) -> Result<(), String> { + let _reader = &self.workspace; + if !due(self.config.as_ref(), &self.store.root, self.settled_at_ms) { + return Err("Recovery retention no longer permits cleanup".into()); + } + unmanaged(&self.store)?; + for owner in &self.owners { + owner.lifetime.verify()?; + self.store.verify_worker_namespace(&owner.origin)?; + owner.intent.verify(&owner.journal, "intent.json")?; + owner.phase.verify( + &self.workers, + owner + .lifetime + .binding() + .source_phase + .as_deref() + .ok_or("Missing paired phase")?, + )?; + } + Ok(()) + } + fn verify(&self) -> Result<(), String> { + self.routes()?; + for owner in &self.owners { + if self.store.worker_digest(Some(&owner.origin))?.as_ref() != Some(&owner.digest) { + return Err("Paired cleanup intent changed".into()); + } + let known = self + .plan + .files + .iter() + .filter(|file| { + file.journal.as_deref() == Some(owner.origin.binding.journal_id.as_str()) + }) + .map(|file| file.name.as_str()) + .chain(std::iter::once("intent.json")) + .collect::>(); + if storage::names(&owner.journal, super::super::ENTRY_LIMIT * 3)? + .iter() + .any(|name| !known.contains(name.as_str())) + { + return Err("Unplanned entry appeared before paired cleanup".into()); + } + } + let (_, parent) = self + .store + .journal(&self.owners[0].origin.binding.journal_id)?; + self.store.verify_settlement(&parent) + } + fn delete(self) -> Result { + self.verify()?; + for file in &self.plan.files { + self.pin(file)?; + } + for owner in &self.owners { + let directory = owner + .journal + .try_clone() + .map_err(storage::fail)? + .into_std_file(); + let mut generation = ownership::generation(&directory)?; + for file in self.plan.files.iter().filter(|file| { + file.journal.as_deref() == Some(owner.origin.binding.journal_id.as_str()) + }) { + self.routes()?; + if ownership::generation(&directory)? != generation { + return Err("Paired cleanup directory changed".into()); + } + if let Some(_pin) = self.pin(file)? { + owner + .journal + .remove_file(&file.name) + .map_err(storage::fail)?; + storage::sync(&owner.journal)?; + generation = ownership::generation(&directory)?; + } + } + } + self.verify()?; + self.plan.total_bytes().map_err(storage::fail) + } + fn pin(&self, file: &Payload) -> Result, String> { + let owner = self + .owners + .iter() + .find(|owner| file.journal.as_deref() == Some(owner.origin.binding.journal_id.as_str())) + .ok_or("Foreign cleanup journal")?; + match owner.journal.symlink_metadata(&file.name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(storage::fail(error)), + _ => {} + } + let pin = storage::open(&owner.journal, Path::new(&file.name))?; + if storage::identity(&pin)? != file.identity + || ownership::generation(&pin)? != file.generation + { + return Err("Paired cleanup payload changed".into()); + } + Ok(Some(pin)) + } +} diff --git a/crates/cyber-snapshot/src/reclamation.rs b/crates/cyber-snapshot/src/reclamation.rs index 82146ceba..5c7034159 100644 --- a/crates/cyber-snapshot/src/reclamation.rs +++ b/crates/cyber-snapshot/src/reclamation.rs @@ -1,13 +1,13 @@ //! Application-owned Store authority for private snapshot maintenance. use cyber_server::runtime::{ - SnapshotOrigin, SnapshotRestoreReceipt, - snapshot_reclamation::{Plan, Record}, + SnapshotOrigin, + snapshot_reclamation::{Plan, Receipt, Record}, }; use std::collections::BTreeSet; #[derive(Debug, Clone)] pub struct Settlement { - pub receipt: SnapshotRestoreReceipt, + pub receipt: Receipt, pub settled_at_ms: i64, } @@ -18,7 +18,7 @@ pub type Collect = Box) -> Result<(), String> + Sen /// retained by that closure. Neither method may acknowledge a native activity. pub trait Policy: Send + Sync { fn receipt(&self, origin: &SnapshotOrigin) -> Result, String>; - fn record(&self, receipt: &SnapshotRestoreReceipt) -> Result, String>; + fn record(&self, receipt: &Receipt) -> Result, String>; fn reclaim(&self, plan: Plan, delete: Delete) -> Result; fn collect(&self, collect: Collect) -> Result<(), String>; fn refused(&self, directory: &str, error: &str); diff --git a/docs/rewind.md b/docs/rewind.md index 483af12ff..83a44b8c0 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -176,8 +176,15 @@ from durable settlement. Disabled snapshots or zero retention preserve payloads. Live readers, independent unknown effects, changed payloads or incomplete native ownership refuse cleanup. A full capture may attempt the same eligible cleanup and retry once. Cleanup retains the intent, lifetime records and durable retry -receipt, and never reads or restores source files. Managed Git, keep/rollback, -preparation-construction and legacy payload reclamation remain unfinished. +receipt, and never reads or restores source files. + +Acknowledged keep/rollback recovery can also reclaim its original and resolution +regular-file stages together under both exited worker lifetimes. It requires the +exact paired receipt, both native acknowledgements, the original settlement link +and unchanged recorded private slots. Both inventories are checked before deletion; +interrupted cleanup resumes only through its durable plan. Both intents and lookup +receipts remain usable afterward. Managed Git, preparation-construction, metadata/ +CAS and legacy payload reclamation remain unfinished. Before file changes, restore durably publishes an intent and prepares private stages. Replacements retain displaced inodes; progress and completion are flushed diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index f77f3fc36..060377a7b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3265,3 +3265,78 @@ Store adapter retain their project-specific dependency roles. The full recovery feature still requires the acceptance gaps above; this is not approval of the complete M6 feature or either issue. All final local checks are recorded above; no PR/push or parent-task completion is claimed. + +## Paired keep/rollback payload reclamation (2026-11-07, local) + +The next M6 path removes original and resolution regular private stages together +only after the exact paired Store receipt, both native acknowledgements and the +parent's settlement link agree. It preserves both intents, immutable final slot +observations, worker/phase metadata, CAS images and lookup receipts. An untagged +internal receipt carrier retains ordinary version-1 serialization and aggregate +IDs; version-2 plans identify each leaf's exact original/resolution journal. Both +complete inventories are preflighted before any unlink, with both exited lifetime +locks, metadata pins and the shared namespace/Location guard retained. The Store +writer re-attests both owners and references around the bounded native callback. + +Original leaves must match the resolution's immutable final private observations; +resolution leaves must match its own displaced originals. Missing leaves require +an already committed matching plan. Resume indexes frozen leaves by journal/name +rather than repeatedly scanning the full plan. Retention uses paired settlement; +zero/disabled settings, unknown native owners and incomplete proof still refuse. +An admitted original is required. A resolution with zero changes, as in keep, +must instead have an exact non-admitted phase with no origin; other resolutions +require admission. This does not generalize ordinary cleanup's positive phase +requirement or admit preparation construction, legacy evidence or managed Git. + +The real CLI fixture kills a service after a partial source restore, performs +reviewed keep and rollback, and now reopens the stopped service's actual Store +for maintenance. It checks young retention, both independently held worker locks, +a durable plan cut before deletion, refusal on an unknown entry before any unlink, +synthetic progress of one planned unlink, complete retry, retained intent bytes, +unchanged later source/symlink/target and idempotent accounting. The plan cut and +single unlink are synthetic states, not additional cleanup process-kill claims. +The fixture restarts the actual service and verifies original request lookup and +explicit acknowledgement still work without inference or transcript movement. + +Code-review skill inspection covered receipt serialization and legacy ID framing, +two-owner attestation, leaf/journal association and bounds, native phase admission, +full preflight, private metadata pins, no-follow unlinks/fsync and retained retry +receipts. The existing application policy owns database transactions; native +snapshot cleanup remains a filesystem adapter with no Store/source replay port. +Clippy identified test helper cognitive complexity 17/15 and 19/15; the fixture +was split by cleanup setup, interrupted-plan verification and lookup responsibilities, +without allowances. Native paired preparation/deletion passes the same threshold. + +Initial local checks: the ordinary App cleanup suite passed all 15 cases in +43.99 seconds, and the real keep/rollback fixture passed in 81.50 seconds before +its final test-helper refactor. HTTP passed all 80 cases, strict OpenSpec passed +117 items and SDK generation consistency passed. Final source checks and the +complete serial CLI composition are recorded below when finished. This path does +not close M6: managed resource/Git cleanup, non-admitted original/preparation and +construction classes, CAS/metadata reclamation and the full native crash, lease +and platform matrix remain required. No PR/push or parent acceptance is claimed. + +Review then tightened consuming-time inventory checks to allow only the durable +plan's leaves, rather than every possible stage index in the journal. A valid +resolution can have an intentionally empty stage (rollback recreating a deleted +file). The strengthened real-kill fixture includes such a file and injects owner +bytes into that empty index inside the consuming writer callback, after native +planning. Cleanup must report the unplanned entry before removing any planned +leaf, retain the injected bytes and resume only after explicit fixture cleanup. +This is a regression for that review correction, not an ownership/admission bypass. + +Final local macOS arm64 evidence: the complete serial CLI branch/recovery run +passed all 18 cases in 346.57 seconds before the final inventory tightening. The +final focused keep/rollback case passed both actions and the strengthened empty-slot +regression in 84.39 seconds afterward; the full 18-case run was not repeated for +that paired-only correction. Ordinary legacy ID/serialization compatibility passed +separately in 5.37 seconds. All-target App/server/snapshot/CLI Clippy passed after +the final correction, with no complexity allowances. Format/diff checks, generated +SDK consistency, strict OpenSpec (117 passed) and spec lint (0 errors/25 warnings) +passed. HTTP's 80 cases passed before that native inventory correction. No full +snapshot-library/workspace/SDK execution, additional native platform or CI run was +performed for this increment. C deprecation/linker unwind warnings were observed +without unchanged-main attribution. All Cargo tests/checks/Clippy used +CARGO_INCREMENTAL=0 and --locked. No unresolved critical/major code-review finding +remains within this paired regular-payload scope; the broader acceptance obligations +listed above remain open and neither issue is complete. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 2fbfff436..d4f1c58e1 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -828,3 +828,25 @@ or rewritten event data and event deletion cascades remove only that event's roo The same complete index protects staged, inactive-branch and saved-version history. A missing/unreadable/oversized index refuses cleanup. Reference lookup at the writer must not repeatedly parse the entire conversation history while other writes wait. + +### Paired recovery payload cleanup + +Acknowledged keep/rollback cleanup SHALL bind the exact durable two-owner recovery +receipt, both immutable native intents and the parent's settlement link. The +reclamation frame SHALL distinguish original and resolution journal leaves while +preserving existing ordinary restore plan identities and serialization. Native +cleanup SHALL retain both exited worker lifetimes, their phase pins and the shared +Location/namespace guards. The initial paired admission requires an admitted original source phase and +empty complete checkout inventories. A resolution with no journal changes must +retain a positively non-admitted phase with no origin; other resolutions require +positive admission; other +preparation/managed resource classes remain required separately. + +Original private slots SHALL match the resolution's immutable final observations; +resolution slots SHALL match its own completed journal's original displaced +images. Both complete inventories SHALL be preflighted before any unlink. Partial +progress SHALL require the committed exact plan and immutable ownership evidence. +Neither source bytes nor current source routes grant cleanup authority or need to +be read. Both intents, settlement links, phase metadata and durable lookup receipts +SHALL survive payload reclamation. Retention SHALL begin at durable paired settlement +and be rechecked during deletion; unknown native owners continue to protect both. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md index 50e2ce232..19ed3d8c4 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md @@ -43,3 +43,11 @@ - **THEN** existing references SHALL be backfilled and each later change SHALL update its event-owned reference rows in the same transaction - **AND** shared references SHALL remain protected until every referencing event is removed - **AND** missing, unreadable or oversized reference views SHALL refuse reclamation + +#### Scenario: Reclaim an acknowledged recovery pair +- **GIVEN** exact paired keep/rollback settlement and matching original/resolution native acknowledgement +- **WHEN** private recovery payloads become eligible after retention +- **THEN** a version-2 reclamation plan SHALL bind both journals and their exact immutable private leaves +- **AND** both exited lifetimes and complete inventories SHALL be checked before any unlink +- **AND** interrupted deletion SHALL resume only from that durable plan without reading or replaying source +- **AND** original version-1 restore plan serialization and IDs SHALL remain unchanged diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index a373d29f8..25723a5cf 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -532,3 +532,12 @@ interrupted plan resume, missing/changed proof, disabled/zero retention and cap pressure before accepting this path. Managed Git, recovery-resolution/preparation construction payload cleanup and complete reclamation/platform acceptance remain required; M6 and all parent tasks stay open. + +M6 paired-payload continuation: reclaim original and resolution regular private +slots together after exact keep/rollback Store and native acknowledgement. Bind +both journals, preserve version-1 ordinary plan IDs, retain both lifetimes/phases, +and preflight complete immutable outcome/displaced inventories before deleting. +Exercise the real partial-restore kill fixture for keep and rollback, both live +worker locks, young retention, unknown entries and planned partial progress. +Construction/preparation payloads, managed resources/Git, CAS/metadata reclamation +and the complete native cut/platform matrix remain required; M6 stays open. From 3e187e2035eb01c0f2bf6c7954862c1a3a420f91 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 11:55:51 -0400 Subject: [PATCH 44/87] Reclaim cancelled prepared snapshot payloads --- crates/cyber-cli/tests/branches_cli.rs | 384 +++++++++++++++++- crates/cyber-cli/tests/support/reclamation.rs | 75 +++- crates/cyber-server/src/runtime/activity.rs | 4 + .../src/runtime/snapshot_lifetime.rs | 8 + .../src/runtime/snapshot_preparation.rs | 47 ++- .../src/runtime/snapshot_reclamation.rs | 121 ++++-- crates/cyber-snapshot/src/copy/lifetime.rs | 34 ++ crates/cyber-snapshot/src/copy/ownership.rs | 3 + crates/cyber-snapshot/src/copy/reclamation.rs | 26 +- .../src/copy/reclamation/cancelled.rs | 201 +++++++++ .../src/copy/reclamation/paired.rs | 1 + docs/rewind.md | 7 + .../implementation.md | 72 ++++ .../managed-recovery.md | 21 + .../specs/storage-events/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 11 + 16 files changed, 962 insertions(+), 60 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/reclamation/cancelled.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 194bdf1b8..2ce45f028 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -806,6 +806,26 @@ fn terminal_preparation_recovery_survives_client_kill_with_a_committed_reply_wit #[cfg(any(target_os = "macos", target_os = "linux"))] fn cancelled_pre_effect_preparation(published: bool, terminal: bool) { + cancelled_preparation(published, terminal, false); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[test] +fn fully_prepared_pre_effect_cancellation_reclaims_only_persisted_target_inodes() { + cancelled_preparation(true, false, true); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +struct PreparationFixture { + model: scripted_model::ScriptedModel, + env: Env, + id: String, + first: String, + messages: Vec, + after: String, +} +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn preparation_fixture(prepared: bool) -> PreparationFixture { let model = scripted_model::ScriptedModel::start(reply); let env = Env::new(&model); let repo = env.root.join("repo"); @@ -820,9 +840,37 @@ fn cancelled_pre_effect_preparation(published: bool, terminal: bool) { for index in 0..1280 { std::fs::write(repo.join(format!("bulk_{index:04}")), &after).unwrap(); } + if prepared { + std::fs::write(repo.join("aa_removed"), "after-only source").unwrap(); + } env.exec(Some(&id), "after snapshot"); let messages = env.messages(&id); - let activity = kill_before_source_admission(&env, &id, &first, published); + PreparationFixture { + model, + env, + id, + first, + messages, + after, + } +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn cancelled_preparation(published: bool, terminal: bool, prepared: bool) { + let PreparationFixture { + model, + env, + id, + first, + messages, + after, + } = preparation_fixture(prepared); + let repo = env.root.join("repo"); + let activity = if prepared { + kill_after_prepared_images(&env, &id, &first) + } else { + kill_before_source_admission(&env, &id, &first, published) + }; let requests = model.requests().len(); let target = env.root.join("preparation-owner-target"); std::fs::write(&target, "later owner bytes").unwrap(); @@ -842,6 +890,9 @@ fn cancelled_pre_effect_preparation(published: bool, terminal: bool) { let request_id = "before-origin/ owner #?%"; let body = json!({"confirmation":review["confirmation"],"request_id":request_id}); assert_preparation_client_refusals(&env, &id, &activity, &body); + if prepared { + assert_prepared_review_digest(&env, &id, &activity, &review, &body); + } let receipt = preparation_with_client(&env, &id, &activity, &body, terminal); let request_id = receipt["request_id"].as_str().unwrap(); let body = json!({"confirmation":receipt["confirmation"],"request_id":request_id}); @@ -855,24 +906,7 @@ fn cancelled_pre_effect_preparation(published: bool, terminal: bool) { if published { assert_preparation_tombstone(&env, &receipt); } - kill_registered_service(&env); - assert_preparation_history(&env, &id, &activity, &receipt); - assert_eq!( - json(&preparation_confirm_cli(&env, &id, &activity, &body)), - receipt - ); - assert_eq!( - json(&preparation_request_cli( - &env, &id, &activity, request_id, false - )), - receipt - ); - assert_eq!( - json(&preparation_request_cli( - &env, &id, &activity, request_id, true - )), - receipt - ); + assert_preparation_receipt_restart(&env, &receipt, &body); assert!( std::fs::symlink_metadata(repo.join("bulk_0000")) .unwrap() @@ -886,9 +920,39 @@ fn cancelled_pre_effect_preparation(published: bool, terminal: bool) { assert_eq!(env.file("bulk_1279"), Some(after)); assert_eq!(env.messages(&id), messages); assert_eq!(model.requests().len(), requests); + if prepared { + assert_preparation_cleanup_lookup(&env, &receipt); + } + assert_eq!(env.messages(&id), messages); + assert_eq!(model.requests().len(), requests); env.exec(Some(&id), "after preparation cancellation"); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_preparation_receipt_restart(env: &Env, receipt: &Value, body: &Value) { + let session = receipt["lifetime"]["owner"]["session_id"].as_str().unwrap(); + let activity = receipt["activity_id"].as_str().unwrap(); + let request_id = receipt["request_id"].as_str().unwrap(); + kill_registered_service(env); + assert_preparation_history(env, session, activity, receipt); + assert_eq!( + json(&preparation_confirm_cli(env, session, activity, body)), + *receipt + ); + assert_eq!( + json(&preparation_request_cli( + env, session, activity, request_id, false + )), + *receipt + ); + assert_eq!( + json(&preparation_request_cli( + env, session, activity, request_id, true + )), + *receipt + ); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn preparation_with_client( env: &Env, @@ -904,6 +968,190 @@ fn preparation_with_client( } } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_preparation_cleanup_lookup(env: &Env, receipt: &Value) { + assert_prepared_payload_cleanup(env, receipt); + let session = receipt["lifetime"]["owner"]["session_id"].as_str().unwrap(); + assert_eq!( + json(&preparation_request_cli( + env, + session, + receipt["activity_id"].as_str().unwrap(), + receipt["request_id"].as_str().unwrap(), + true + )), + *receipt + ); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_prepared_review_digest( + env: &Env, + session: &str, + activity: &str, + review: &Value, + body: &Value, +) { + let journal = review["origin"]["binding"]["journal_id"].as_str().unwrap(); + let path = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .map(|entry| { + entry + .unwrap() + .path() + .join("journals") + .join(journal) + .join("intent.json") + }) + .find(|path| path.exists()) + .unwrap(); + let bytes = std::fs::read(&path).unwrap(); + let mut changed: Value = serde_json::from_slice(&bytes).unwrap(); + changed["created"] = (changed["created"].as_u64().unwrap() + 1).into(); + std::fs::write(&path, changed.to_string()).unwrap(); + let refused = preparation_confirm_cli(env, session, activity, body); + assert_eq!(refused.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&refused.stderr).contains("review changed"), + "{}", + String::from_utf8_lossy(&refused.stderr) + ); + std::fs::write(path, bytes).unwrap(); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_prepared_payload_cleanup(env: &Env, reply: &Value) { + use cyber_server::runtime::{SnapshotPreparationReceipt, snapshot_reclamation::Receipt}; + exits(&env.cyber(&["service", "stop"]), 0); + let maintenance = reclamation_support::Maintenance::open(&env.root.join("cyber-home")); + let receipt: SnapshotPreparationReceipt = serde_json::from_value(reply.clone()).unwrap(); + let journal = maintenance + .namespace + .join("journals") + .join(&receipt.origin.as_ref().unwrap().binding.journal_id); + let intent = std::fs::read(journal.join("intent.json")).unwrap(); + let cancellation = std::fs::read(journal.join("preparation.json")).unwrap(); + let source = env.root.join("repo"); + assert_eq!(maintenance.reclaim(&source).unwrap(), 0); + maintenance.age(receipt.activity_id.clone()); + let receipt = Receipt::Preparation(Box::new(receipt)); + assert_prepared_inode_and_lifetime_guards(&maintenance, &source, &journal, &receipt); + assert_prepared_store_receipt_binding(&maintenance, &source, &journal, &receipt); + assert_prepared_interrupted_plan(&maintenance, &source, &journal, &receipt); + assert!(maintenance.reclaim(&source).unwrap() > 0); + assert!( + maintenance + .policy + .record(&receipt) + .unwrap() + .unwrap() + .completed + ); + assert_eq!(maintenance.reclaim(&source).unwrap(), 0); + assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); + assert_eq!( + std::fs::read(journal.join("preparation.json")).unwrap(), + cancellation + ); + assert_eq!(std::fs::read_dir(&journal).unwrap().count(), 2); + assert!( + std::fs::symlink_metadata(source.join("bulk_0000")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_to_string(env.root.join("preparation-owner-target")).unwrap(), + "later owner bytes" + ); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_prepared_interrupted_plan( + maintenance: &reclamation_support::Maintenance, + source: &std::path::Path, + journal: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + assert!(maintenance.plan_only(source).is_err()); + let pending = maintenance.policy.record(receipt).unwrap().unwrap(); + assert!(!pending.completed); + assert_eq!(pending.plan.version, 3); + assert!(pending.plan.intent_digest.is_some()); + let unknown = journal.join("foreign-evidence"); + std::fs::write(&unknown, b"owner bytes").unwrap(); + assert!(maintenance.reclaim(source).is_err()); + let first = journal.join(&pending.plan.files[0].name); + assert!(first.exists()); + std::fs::remove_file(unknown).unwrap(); + // Synthetic progress of one planned private unlink, following a real kill. + std::fs::remove_file(first).unwrap(); + std::fs::File::open(journal).unwrap().sync_all().unwrap(); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_prepared_store_receipt_binding( + maintenance: &reclamation_support::Maintenance, + source: &std::path::Path, + journal: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + let owner = receipt.origin().unwrap().activity_id.clone(); + let restored_owner = owner.clone(); + let original = maintenance.store.transaction(move |db| { + let raw: String = db.query_row("SELECT data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", [&owner], |row|row.get(0))?; + let mut legacy: Value = serde_json::from_str(&raw).unwrap(); + assert!(legacy.as_object_mut().unwrap().remove("snapshot_preparation_intent_digest").is_some()); + db.execute("UPDATE event SET data=?1 WHERE aggregate_id=?2 AND seq=(SELECT MAX(seq) FROM event WHERE aggregate_id=?2)", rusqlite::params![legacy.to_string(), owner])?; + Ok(raw) + }).unwrap(); + let error = maintenance.reclaim(source).unwrap_err(); + assert!(error.contains("committed native intent digest"), "{error}"); + maintenance.store.transaction(move |db| { + db.execute("UPDATE event SET data=?1 WHERE aggregate_id=?2 AND seq=(SELECT MAX(seq) FROM event WHERE aggregate_id=?2)", rusqlite::params![original,restored_owner])?; + Ok(()) + }).unwrap(); + let path = journal.join("intent.json"); + let bytes = std::fs::read(&path).unwrap(); + let mut changed: Value = serde_json::from_slice(&bytes).unwrap(); + changed["created"] = (changed["created"].as_u64().unwrap() + 1).into(); + std::fs::write(&path, changed.to_string()).unwrap(); + let error = maintenance.reclaim(source).unwrap_err(); + assert!(error.contains("committed native intent digest"), "{error}"); + assert!(maintenance.policy.record(receipt).unwrap().is_none()); + std::fs::write(path, bytes).unwrap(); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_prepared_inode_and_lifetime_guards( + maintenance: &reclamation_support::Maintenance, + source: &std::path::Path, + journal: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + let proof = receipt.owners().unwrap().remove(0); + let held = std::fs::File::open( + maintenance + .namespace + .join("workers") + .join(&proof.origin.activity_id), + ) + .unwrap(); + held.try_lock().unwrap(); + assert!(maintenance.reclaim(source).is_err()); + drop(held); + let stage = journal.join("stage_1"); + let retained = journal.join("retained-original"); + let bytes = std::fs::read(&stage).unwrap(); + std::fs::rename(&stage, &retained).unwrap(); + std::fs::write(&stage, bytes).unwrap(); + let error = maintenance.reclaim(source).unwrap_err(); + assert!(error.contains("persisted ownership"), "{error}"); + assert!(maintenance.policy.record(receipt).unwrap().is_none()); + std::fs::remove_file(&stage).unwrap(); + std::fs::rename(retained, stage).unwrap(); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn terminal_preparation(env: &Env, session: &str, activity: &str) -> Value { let template = env.command(&[]); @@ -1087,7 +1335,12 @@ fn assert_preparation_tombstone(env: &Env, receipt: &Value) { if let Ok(bytes) = std::fs::read(journal.join("intent.json")) { let intent: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(intent["completed"], false); - assert_eq!(intent["acknowledged"], false); + assert!( + !intent + .get("acknowledged") + .and_then(Value::as_bool) + .unwrap_or(false) + ); assert!(intent["applied"].as_array().unwrap().is_empty()); } } @@ -1152,6 +1405,94 @@ fn kill_before_source_admission(env: &Env, id: &str, first: &str, published: boo activity } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn kill_after_prepared_images(env: &Env, id: &str, first: &str) -> String { + use rusqlite::OptionalExtension; + let selection = reviewed_selection(env, first, "both", "commit"); + let preview = reviewed_preview(env, id, &selection); + let mut body = selection; + body["confirmation"] = preview["confirmation"].clone(); + let mut pending = env + .command(&[ + "api", + "POST", + &format!("/sessions/{id}/rewind/confirm"), + "-d", + &body.to_string(), + ]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let data = env.root.join("cyber-home/data"); + let database = std::fs::read_dir(&data) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|extension| extension == "db")) + .unwrap(); + let db = rusqlite::Connection::open(database).unwrap(); + db.busy_timeout(Duration::from_secs(5)).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(45); + let activity = loop { + let activity: Option = db.query_row("SELECT aggregate_id FROM event e WHERE type='native.activity.changed.1' AND seq=1 AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_origin') IS NOT NULL AND json_extract(data,'$.snapshot_source_admitted')=0 AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq)", [id], |row|row.get(0)).optional().unwrap(); + if let Some(activity) = activity { + break activity; + } + assert!( + pending.try_wait().unwrap().is_none() && std::time::Instant::now() < deadline, + "preparation origin was not observed" + ); + std::thread::sleep(Duration::from_millis(1)); + }; + // Real SQLite writer contention holds source admission, not preparation I/O. + // No row is edited and no native acknowledgement or effect hook is replaced. + db.execute_batch("BEGIN IMMEDIATE").unwrap(); + let raw: String = db + .query_row( + "SELECT data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + [&activity], + |row| row.get(0), + ) + .unwrap(); + let record: Value = serde_json::from_str(&raw).unwrap(); + assert_eq!(record["snapshot_source_admitted"], false); + let journal_id = record["snapshot_origin"]["binding"]["journal_id"] + .as_str() + .unwrap(); + let namespace = std::fs::read_dir(data.join("snapshot/global")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.join("journals").join(journal_id).exists()) + .unwrap(); + let intent = namespace + .join("journals") + .join(journal_id) + .join("intent.json"); + loop { + let prepared = std::fs::read(&intent) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|intent| { + intent["ownership"].is_object() + && intent["applied"].as_array().is_some_and(Vec::is_empty) + }); + if prepared { + break; + } + assert!( + pending.try_wait().unwrap().is_none() && std::time::Instant::now() < deadline, + "prepared identities were not published" + ); + std::thread::sleep(Duration::from_millis(2)); + } + kill_registered_service(env); + assert!(!pending.wait_with_output().unwrap().status.success()); + db.execute_batch("ROLLBACK").unwrap(); + let admitted: bool = db.query_row("SELECT json_extract(data,'$.snapshot_source_admitted') FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", [&activity], |row|row.get(0)).unwrap(); + assert!(!admitted, "kill crossed the source-admission boundary"); + activity +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn recovery_confirm_cli(env: &Env, id: &str, request: &Value) -> Output { env.cyber(&[ @@ -1573,7 +1914,8 @@ fn assert_paired_live_owners_refuse( namespace: &std::path::Path, receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, ) { - for (origin, _) in receipt.owners().unwrap() { + for proof in receipt.owners().unwrap() { + let origin = proof.origin; let held = std::fs::File::open(namespace.join("workers").join(&origin.activity_id)).unwrap(); held.try_lock().unwrap(); diff --git a/crates/cyber-cli/tests/support/reclamation.rs b/crates/cyber-cli/tests/support/reclamation.rs index d2a5e4978..713d5fd21 100644 --- a/crates/cyber-cli/tests/support/reclamation.rs +++ b/crates/cyber-cli/tests/support/reclamation.rs @@ -5,7 +5,7 @@ use cyber_server::runtime::{ }; use cyber_snapshot::reclamation::{Collect, Delete, Policy, Settlement}; use std::{ - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, Mutex}, }; @@ -38,3 +38,76 @@ impl Policy for Planned { self.0.refused(directory, error); } } + +pub struct Maintenance { + pub store: Arc, + pub policy: Arc, + pub namespace: PathBuf, + root: PathBuf, + runtime: tokio::runtime::Runtime, + snapshots: Arc, +} +impl Maintenance { + pub fn open(root: &Path) -> Self { + let data = root.join("data"); + let database = std::fs::read_dir(&data) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|extension| extension == "db")) + .unwrap(); + let store = Arc::new( + cyber_store::Store::open(cyber_store::StoreOptions::new( + cyber_core::paths::DatabaseLocation::File(database), + cyber_server::runtime::Runtime::registry(), + )) + .unwrap(), + ); + let policy = cyber_app::snapshot_reclamation::policy(store.clone()); + let namespace = std::fs::read_dir(data.join("snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let snapshots = Self::snapshots(root, policy.clone()); + Self { + store, + policy, + namespace, + root: root.into(), + runtime: tokio::runtime::Runtime::new().unwrap(), + snapshots, + } + } + fn snapshots(root: &Path, policy: Arc) -> Arc { + cyber_snapshot::GitSnapshots::with_reclamation( + root.join("data"), + Arc::new(|_| serde_json::json!({"snapshots":{"retention_days":7}})), + vec![root.join("config"), root.join("state"), root.join("cache")], + None, + Some(policy), + ) + } + pub fn reclaim(&self, source: &Path) -> Result { + self.runtime + .block_on(self.snapshots.reclaim(source.to_str().unwrap())) + } + pub fn plan_only(&self, source: &Path) -> Result { + let cut = Self::snapshots( + &self.root, + Arc::new(Planned(self.policy.clone(), Arc::new(Mutex::new(None)))), + ); + self.runtime.block_on(cut.reclaim(source.to_str().unwrap())) + } + pub fn age(&self, activity: String) { + self.store + .transaction(move |db| { + db.execute( + "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", + rusqlite::params![8i64 * 86400000, activity], + )?; + Ok(()) + }) + .unwrap(); + } +} diff --git a/crates/cyber-server/src/runtime/activity.rs b/crates/cyber-server/src/runtime/activity.rs index bb87b3d60..f6f69b7d4 100644 --- a/crates/cyber-server/src/runtime/activity.rs +++ b/crates/cyber-server/src/runtime/activity.rs @@ -40,6 +40,8 @@ pub(super) struct Record { #[serde(default, skip_serializing_if = "Option::is_none")] pub snapshot_preparation_receipt: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_preparation_intent_digest: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub snapshot_source_admitted: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_bindings: Option>, @@ -140,6 +142,7 @@ impl Scope { snapshot_source_admitted: lifetime.as_ref().map(|_| false), snapshot_lifetime: lifetime, snapshot_preparation_receipt: None, + snapshot_preparation_intent_digest: None, admission_bindings: Some(authority.bindings().to_vec()), }; let captured = authority.clone(); @@ -210,6 +213,7 @@ impl Scope { snapshot_source_admitted: lifetime.as_ref().map(|_| false), snapshot_lifetime: lifetime, snapshot_preparation_receipt: None, + snapshot_preparation_intent_digest: None, admission_bindings: Some(authority.bindings.clone()), }; inner diff --git a/crates/cyber-server/src/runtime/snapshot_lifetime.rs b/crates/cyber-server/src/runtime/snapshot_lifetime.rs index 26fc59ba1..234c13fc2 100644 --- a/crates/cyber-server/src/runtime/snapshot_lifetime.rs +++ b/crates/cyber-server/src/runtime/snapshot_lifetime.rs @@ -169,6 +169,14 @@ pub trait SnapshotLifetime: Send { fn verify_preparation(&self, _origin: Option<&super::SnapshotOrigin>) -> Result<(), String> { Err("Native preparation phase proof is unavailable".into()) } + /// Read only a fully prepared native intent; absence grants no payload cleanup. + /// This metadata-only operation must not reenter the primary Store. + fn preparation_intent_digest( + &self, + _origin: Option<&super::SnapshotOrigin>, + ) -> Result, String> { + Ok(None) + } /// Read-only candidate check against retained native preparation receipts. /// Called at the primary writer; no primary Store reentry or metadata writes. fn prepare_preparation_acknowledgement( diff --git a/crates/cyber-server/src/runtime/snapshot_preparation.rs b/crates/cyber-server/src/runtime/snapshot_preparation.rs index 9ae8879ec..ef6a23e3d 100644 --- a/crates/cyber-server/src/runtime/snapshot_preparation.rs +++ b/crates/cyber-server/src/runtime/snapshot_preparation.rs @@ -305,6 +305,30 @@ fn acknowledged_target( }) } +pub(super) fn verify_committed( + db: &Connection, + receipt: &SnapshotPreparationReceipt, +) -> Result<(), StoreError> { + acknowledged_target(db, receipt).map(|_| ()) +} + +pub(super) fn verify_reclamation( + db: &Connection, + receipt: &SnapshotPreparationReceipt, + intent_digest: &str, +) -> Result<(), StoreError> { + let target = acknowledged_target(db, receipt)?; + if intent_digest.len() != 64 + || !intent_digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + || target.record.snapshot_preparation_intent_digest.as_deref() != Some(intent_digest) + { + return Err(invalid( + "Preparation cleanup lacks its committed native intent digest", + )); + } + Ok(()) +} + fn request_owner(created: &str, request: &str) -> String { use sha2::{Digest, Sha256}; let bytes = serde_json::to_vec(&("snapshot-preparation-request-v1", created, request)) @@ -344,6 +368,7 @@ struct Worker { resources: Option>, checkouts: Option>, proof: Box, + intent_digest: Option, _lifecycle: tokio::sync::OwnedRwLockReadGuard<()>, _admission: Admission, _completion: Completion, @@ -402,12 +427,13 @@ impl Worker { fn fingerprint(&self) -> Result { use sha2::{Digest, Sha256}; let bytes = serde_json::to_vec(&( - "snapshot-preparation-review-v2", + "snapshot-preparation-review-v3", self.sequence, &self.authority, self.target.sequence, &self.target.raw, self.resources_digest()?, + &self.intent_digest, )) .map_err(|error| RuntimeError::Invalid(error.to_string()))?; Ok(format!("{:x}", Sha256::digest(bytes))) @@ -420,6 +446,16 @@ impl Worker { .verify_preparation(self.target.record.snapshot_origin.as_ref()) .map_err(|error| invalid(&error))?; self.verify_resources().map_err(|error| invalid(&error))?; + if let Some(expected) = &self.intent_digest + && self + .proof + .preparation_intent_digest(self.target.record.snapshot_origin.as_ref()) + .map_err(|error| invalid(&error))? + .as_ref() + != Some(expected) + { + return Err(invalid("Prepared intent changed after cancellation review")); + } let owner = &self.target.binding.owner; let current = match self.target.record.snapshot_preparation_receipt.as_ref() { Some(receipt) => acknowledged_target(db, receipt)?, @@ -465,6 +501,7 @@ impl Worker { let mut record = self.target.record.clone(); record.status = "settled".into(); record.phase = PHASE.into(); + record.snapshot_preparation_intent_digest = self.intent_digest.clone(); record.admission_bindings = None; record.snapshot_preparation_receipt = Some(receipt.clone()); let events = [ @@ -665,6 +702,13 @@ impl Runtime { .map_err(RuntimeError::Conflict) }) .transpose()?; + let intent_digest = if target.record.snapshot_preparation_receipt.is_some() { + target.record.snapshot_preparation_intent_digest.clone() + } else { + proof + .preparation_intent_digest(target.record.snapshot_origin.as_ref()) + .map_err(RuntimeError::Conflict)? + }; let worker = Worker { runtime: self.clone(), handle, @@ -675,6 +719,7 @@ impl Runtime { resources, checkouts, proof, + intent_digest, _lifecycle: lifecycle, _admission: admission, _completion: completion, diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation.rs b/crates/cyber-server/src/runtime/snapshot_reclamation.rs index d32626ecb..85acbfbdb 100644 --- a/crates/cyber-server/src/runtime/snapshot_reclamation.rs +++ b/crates/cyber-server/src/runtime/snapshot_reclamation.rs @@ -1,7 +1,7 @@ //! Durable private-payload cleanup plans. These never settle native activities. use super::{ - SnapshotOrigin, SnapshotRecoveryReceipt, SnapshotRestoreCheckout, SnapshotRestoreReceipt, - activity, + SnapshotLifetimeBinding, SnapshotOrigin, SnapshotPreparationReceipt, SnapshotRecoveryReceipt, + SnapshotRestoreReceipt, activity, }; use cyber_store::{EventRegistry, StoreError, StoredEvent}; use rusqlite::{Connection, OptionalExtension}; @@ -20,6 +20,7 @@ const BYTES: usize = 8 * 1024 * 1024; pub enum Receipt { Restore(SnapshotRestoreReceipt), Recovery(SnapshotRecoveryReceipt), + Preparation(Box), } impl From for Receipt { fn from(value: SnapshotRestoreReceipt) -> Self { @@ -31,10 +32,24 @@ impl From for Receipt { Self::Recovery(value) } } +impl From for Receipt { + fn from(value: SnapshotPreparationReceipt) -> Self { + Self::Preparation(Box::new(value)) + } +} +pub struct OwnerEvidence<'a> { + pub origin: &'a SnapshotOrigin, + pub lifetime: &'a SnapshotLifetimeBinding, + pub resources_digest: &'a str, +} impl Receipt { pub fn origin(&self) -> Result<&SnapshotOrigin, StoreError> { match self { Self::Restore(receipt) => Ok(&receipt.origin), + Self::Preparation(receipt) => receipt + .origin + .as_ref() + .ok_or_else(|| invalid("Preparation cleanup needs a journal origin")), Self::Recovery(receipt) => receipt .checkout .as_ref() @@ -42,17 +57,19 @@ impl Receipt { .ok_or_else(|| invalid("Missing recovery checkout")), } } - pub fn owners(&self) -> Result, StoreError> { + pub fn owners(&self) -> Result>, StoreError> { match self { Self::Restore(receipt) => { receipt.checkout_acknowledgement_digest().map_err(invalid)?; - Ok(vec![( - &receipt.origin, - receipt - .checkout - .as_deref() - .ok_or_else(|| invalid("Missing restore checkout"))?, - )]) + let checkout = receipt + .checkout + .as_ref() + .ok_or_else(|| invalid("Missing restore checkout"))?; + Ok(vec![OwnerEvidence { + origin: &receipt.origin, + lifetime: &checkout.lifetime, + resources_digest: &checkout.resources_digest, + }]) } Self::Recovery(receipt) => { receipt.checkout_acknowledgement_digest().map_err(invalid)?; @@ -60,22 +77,42 @@ impl Receipt { .checkout .as_ref() .ok_or_else(|| invalid("Missing recovery checkout"))?; + let original = checkout + .original + .as_ref() + .ok_or_else(|| invalid("Missing original recovery checkout"))?; Ok(vec![ - ( - &receipt.original, - checkout - .original - .as_deref() - .ok_or_else(|| invalid("Missing original recovery checkout"))?, - ), - (&checkout.resolution_origin, &checkout.resolution), + OwnerEvidence { + origin: &receipt.original, + lifetime: &original.lifetime, + resources_digest: &original.resources_digest, + }, + OwnerEvidence { + origin: &checkout.resolution_origin, + lifetime: &checkout.resolution.lifetime, + resources_digest: &checkout.resolution.resources_digest, + }, ]) } + Self::Preparation(receipt) => { + receipt.checkout_acknowledgement_digest().map_err(invalid)?; + Ok(vec![OwnerEvidence { + origin: self.origin()?, + lifetime: &receipt.lifetime, + resources_digest: receipt + .checkout_resources_digest + .as_deref() + .ok_or_else(|| invalid("Missing preparation checkout digest"))?, + }]) + } } } fn verify(&self, db: &Connection) -> Result<(), StoreError> { match self { Self::Restore(receipt) => super::snapshot_restore_receipt::verify(db, receipt), + Self::Preparation(receipt) => { + super::snapshot_preparation::verify_committed(db, receipt) + } Self::Recovery(receipt) => { super::snapshot_acknowledgement::verify_committed(db, receipt) } @@ -97,6 +134,8 @@ pub struct File { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Plan { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub intent_digest: Option, pub version: u32, pub receipt: Receipt, pub files: Vec, @@ -127,10 +166,10 @@ impl Plan { } pub fn validate(&self) -> Result<(), StoreError> { let owners = self.receipt.owners()?; - for (origin, checkout) in &owners { - if origin.worktree_id.is_some() - || checkout.resources_digest - != checkout + for owner in &owners { + if owner.origin.worktree_id.is_some() + || owner.resources_digest + != owner .lifetime .checkout_resources_digest(&[]) .map_err(invalid)? @@ -143,8 +182,16 @@ impl Plan { let version = match &self.receipt { Receipt::Restore(_) => 1, Receipt::Recovery(_) => 2, + Receipt::Preparation(_) => 3, }; if self.version != version + || match &self.receipt { + Receipt::Preparation(_) => self + .intent_digest + .as_deref() + .is_none_or(|value| !hash(value)), + _ => self.intent_digest.is_some(), + } || self.files.len() > LIMIT * owners.len() || serde_json::to_vec(self) .map_err(|e| invalid(e.to_string()))? @@ -164,9 +211,9 @@ impl Plan { if index.is_none_or(|index| index >= LIMIT || file.name != format!("stage_{index}")) || !names.insert((&file.journal, &file.name)) || match &self.receipt { - Receipt::Restore(_) => file.journal.is_some(), - Receipt::Recovery(_) => !owners.iter().any(|(origin, _)| { - file.journal.as_deref() == Some(origin.binding.journal_id.as_str()) + Receipt::Restore(_) | Receipt::Preparation(_) => file.journal.is_some(), + Receipt::Recovery(_) => !owners.iter().any(|owner| { + file.journal.as_deref() == Some(owner.origin.binding.journal_id.as_str()) }), } || file.identity.is_empty() @@ -191,6 +238,15 @@ impl Plan { pub fn authorize(&self, db: &Connection) -> Result, StoreError> { self.validate()?; self.receipt.verify(db)?; + if let Receipt::Preparation(receipt) = &self.receipt { + super::snapshot_preparation::verify_reclamation( + db, + receipt, + self.intent_digest + .as_deref() + .ok_or_else(|| invalid("Missing preparation intent digest"))?, + )?; + } let origin = self.receipt.origin()?; let journal = &origin.binding.journal_id; let directory = origin @@ -200,7 +256,7 @@ impl Plan { .ok_or_else(|| invalid("Non-UTF8 reclamation Location"))?; let protected: bool = db.query_row( "SELECT EXISTS(SELECT 1 FROM event e LEFT JOIN session s ON s.id=json_extract(e.data,'$.session_id') WHERE e.type='native.activity.changed.1' AND coalesce(json_extract(e.data,'$.status'),'unknown')<>'settled' AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq) AND (s.id IS NULL OR (coalesce(json_extract(e.data,'$.status'),'unknown')='unknown' AND json_extract(e.data,'$.snapshot_lifetime') IS NULL) OR s.directory=?2 OR json_extract(e.data,'$.session_id')=?3 OR json_extract(e.data,'$.snapshot_lifetime.owner.directory')=?2 OR EXISTS(SELECT 1 FROM json_tree(e.data) j WHERE j.key='journal_id' AND j.value IN (?1,?4))))", - rusqlite::params![journal, directory, origin.session_id, self.receipt.owners()?[0].0.binding.journal_id], |row| row.get(0), + rusqlite::params![journal, directory, origin.session_id, self.receipt.owners()?[0].origin.binding.journal_id], |row| row.get(0), )?; if protected { return Err(invalid( @@ -242,10 +298,15 @@ pub fn settlement_receipt( }; let record: activity::Record = serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; - let receipt = match (record.snapshot_restore_receipt, record.snapshot_receipt) { - (Some(receipt), None) => Receipt::Restore(receipt), - (None, Some(receipt)) => Receipt::Recovery(receipt), - (None, None) => return Ok(None), + let receipt = match ( + record.snapshot_restore_receipt, + record.snapshot_receipt, + record.snapshot_preparation_receipt, + ) { + (Some(receipt), None, None) => Receipt::Restore(receipt), + (None, Some(receipt), None) => Receipt::Recovery(receipt), + (None, None, Some(receipt)) => Receipt::Preparation(Box::new(receipt)), + (None, None, None) => return Ok(None), _ => return Err(invalid("Ambiguous cleanup receipt")), }; receipt.owners()?; diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index 191963c24..2d263d145 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -1,5 +1,6 @@ //! A worker-specific native lock precedes every durable runtime reservation. use super::{Options, Store, storage}; +use cap_fs_ext::DirExt; use cyber_server::runtime::{SnapshotLifetime, SnapshotLifetimeBinding, SnapshotLifetimeRequest}; use std::{ fs::File, @@ -119,6 +120,39 @@ impl SnapshotLifetime for Lifetime { .ok_or("Missing source-admission namespace")?; store.record_source_admission(&self.binding, origin) } + fn preparation_intent_digest( + &self, + origin: Option<&cyber_server::runtime::SnapshotOrigin>, + ) -> Result, String> { + self.verify_preparation(origin)?; + let Some(origin) = origin else { + return Ok(None); + }; + let store = Store::existing( + &self.data, + &self.binding.owner.directory.to_string_lossy(), + self.options.clone(), + )? + .ok_or("Missing preparation namespace")?; + let directory = store + .journals + .open_dir_nofollow(&origin.binding.journal_id) + .map_err(storage::fail)?; + match directory.symlink_metadata("intent.json") { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(storage::fail(error)), + _ => {} + } + if store + .journal(&origin.binding.journal_id)? + .1 + .ownership + .is_none() + { + return Ok(None); + } + store.worker_digest(Some(origin)) + } fn prepare_preparation_acknowledgement( &self, receipt: &cyber_server::runtime::SnapshotPreparationReceipt, diff --git a/crates/cyber-snapshot/src/copy/ownership.rs b/crates/cyber-snapshot/src/copy/ownership.rs index d99acfc4f..c4c5cb02a 100644 --- a/crates/cyber-snapshot/src/copy/ownership.rs +++ b/crates/cyber-snapshot/src/copy/ownership.rs @@ -482,6 +482,9 @@ impl Ownership { store.verify_root()?; Ok(Observations { files, parents }) } + pub(super) fn prepared_identity(&self, index: usize) -> Option<&str> { + self.files.get(index).and_then(|file| file.stage.as_deref()) + } pub(super) fn original_identity(&self, index: usize) -> Option<&str> { self.files .get(index) diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index 3528ff820..4d625fe81 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -1,4 +1,5 @@ //! Reclaim only exact acknowledged displaced images; keep intent/lifetime receipts. +mod cancelled; mod paired; use super::{ Store, ownership, @@ -22,6 +23,7 @@ struct Prepared { store: Store, journal: Dir, intent: MetadataPin, + cancellation: Option, phase: MetadataPin, workers: Dir, plan: Plan, @@ -130,7 +132,7 @@ pub(crate) fn locations(data: &Path) -> Result, String> { )?) .map_err(storage::fail)?, }; - if ordinary(&record) || paired::eligible(&record) { + if ordinary(&record) || paired::eligible(&record) || cancelled::eligible(&record) { result.insert( record .location @@ -151,8 +153,8 @@ pub(crate) fn reclaim( policy: Arc, id: &str, ) -> Result { - if store.preparation_cancellation(id)?.is_some() { - return Ok(0); + if let Some(receipt) = store.preparation_cancellation(id)? { + return cancelled::reclaim(store, workspace, config, policy, receipt); } let (directory, record) = store.journal(id)?; if paired::eligible(&record) { @@ -280,6 +282,7 @@ fn prepare( let intent_digest = restore.intent_digest.clone(); let plan = Plan { version: 1, + intent_digest: None, receipt, files, }; @@ -298,9 +301,7 @@ fn prepare( store.verify_reclamation_phase( binding, plan.receipt.origin().map_err(storage::fail)?, - &plan.receipt.owners().map_err(storage::fail)?[0] - .1 - .resources_digest, + plan.receipt.owners().map_err(storage::fail)?[0].resources_digest, )?; phase.verify(&workers, name)?; storage::existing_child( @@ -317,6 +318,7 @@ fn prepare( store, journal, intent, + cancellation: None, phase, workers, plan, @@ -346,6 +348,9 @@ impl Prepared { return Err("Snapshot retention no longer permits reclamation".into()); } self.intent.verify(&self.journal, "intent.json")?; + if let Some(pin) = &self.cancellation { + pin.verify(&self.journal, super::preparation::FILE)?; + } self.phase.verify( &self.workers, self.lifetime @@ -367,7 +372,13 @@ impl Prepared { .binding .journal_id, )?; - if !ordinary(&record) + let eligible = match &self.plan.receipt { + cyber_server::runtime::snapshot_reclamation::Receipt::Preparation(receipt) => { + cancelled::verify(&self.store, receipt, &record)? + } + _ => ordinary(&record), + }; + if !eligible || self .store .worker_digest(Some(self.plan.receipt.origin().map_err(storage::fail)?))? @@ -382,6 +393,7 @@ impl Prepared { .iter() .map(|file| file.name.as_str()) .chain(std::iter::once("intent.json")) + .chain(self.cancellation.iter().map(|_| super::preparation::FILE)) .collect::>(); if storage::names(&self.journal, super::ENTRY_LIMIT * 3)? .iter() diff --git a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs new file mode 100644 index 000000000..5736e1fbb --- /dev/null +++ b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs @@ -0,0 +1,201 @@ +//! Reclaim only persisted, fully prepared target inodes after cancellation. +use super::*; +use cyber_server::runtime::{SnapshotPreparationReceipt, snapshot_reclamation::Receipt}; + +pub(super) fn eligible(record: &Journal) -> bool { + record.version >= 3 + && record.origin.is_some() + && record.ownership.is_some() + && record.applied.is_empty() + && !record.acknowledged + && record.settled_by.is_none() + && (!record.completed || record.changes.is_empty()) + && record + .context + .as_ref() + .is_some_and(|context| context.git.is_none()) + && record.backend != Some(JournalBackend::Git) +} +pub(super) fn verify( + store: &Store, + receipt: &SnapshotPreparationReceipt, + record: &Journal, +) -> Result { + let origin = receipt + .origin + .as_ref() + .ok_or("Preparation cleanup has no journal origin")?; + Ok(eligible(record) + && record.origin.as_ref() == Some(origin) + && store + .preparation_cancellation(&origin.binding.journal_id)? + .as_ref() + == Some(receipt)) +} +pub(super) fn reclaim( + store: Store, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + policy: Arc, + receipt: SnapshotPreparationReceipt, +) -> Result { + let origin = receipt + .origin + .as_ref() + .ok_or("Preparation cleanup has no journal origin")?; + let (journal, record) = store.journal(&origin.binding.journal_id)?; + if !verify(&store, &receipt, &record)? { + return Ok(0); + } + let Some(settlement) = policy.receipt(origin)? else { + return Ok(0); + }; + if settlement.receipt != Receipt::Preparation(Box::new(receipt.clone())) { + return Err("Foreign cancellation cleanup receipt".into()); + } + if !due(config.as_ref(), &store.root, settlement.settled_at_ms) { + return Ok(0); + } + let existing = policy.record(&settlement.receipt)?; + if existing.as_ref().is_some_and(|record| record.completed) { + return Ok(0); + } + let prepared = prepare( + store, + workspace, + config, + journal, + record, + settlement, + existing.map(|record| record.plan), + )?; + if prepared.plan.files.is_empty() { + return Ok(0); + } + policy.reclaim(prepared.plan.clone(), Box::new(move || prepared.delete())) +} +fn prepare( + store: Store, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + journal: Dir, + record: Journal, + settlement: Settlement, + frozen: Option, +) -> Result { + let Receipt::Preparation(receipt) = &settlement.receipt else { + return Err("Missing preparation receipt".into()); + }; + receipt.checkout_acknowledgement_digest()?; + let origin = receipt + .origin + .as_ref() + .ok_or("Missing preparation journal origin")?; + store.verify_worker_namespace(origin)?; + let lifetime = store.lock_lifetime(&receipt.lifetime)?; + let resources = lifetime + .checkout_bindings()? + .ok_or("Missing preparation checkout inventory")?; + let digest = receipt + .checkout_resources_digest + .as_deref() + .ok_or("Missing consumed preparation resources")?; + if !resources.is_empty() || receipt.lifetime.checkout_resources_digest(&resources)? != digest { + return Err("Managed preparation cleanup requires resource guards".into()); + } + unmanaged(&store)?; + let workers = storage::existing_child(&store.namespace, "workers", true)?; + let phase_name = receipt + .lifetime + .source_phase + .as_deref() + .ok_or("Missing preparation phase")?; + let phase = MetadataPin::open(&workers, phase_name)?; + store.verify_reclamation_phase_admission(&receipt.lifetime, origin, digest, false)?; + phase.verify(&workers, phase_name)?; + let intent_digest = store + .worker_digest(Some(origin))? + .ok_or("Missing prepared intent digest")?; + let files = payloads(&store, &journal, &record, frozen.as_ref())?; + let intent = MetadataPin::open(&journal, "intent.json")?; + let cancellation = Some(MetadataPin::open( + &journal, + super::super::preparation::FILE, + )?); + let plan = Plan { + version: 3, + intent_digest: Some(intent_digest.clone()), + receipt: settlement.receipt, + files, + }; + plan.validate().map_err(storage::fail)?; + if frozen.as_ref().is_some_and(|frozen| frozen != &plan) { + return Err("Preparation cleanup differs from retained plan".into()); + } + let prepared = Prepared { + store, + journal, + intent, + cancellation, + phase, + workers, + plan, + lifetime, + workspace, + config, + settled_at_ms: settlement.settled_at_ms, + intent_digest, + }; + prepared.verify()?; + Ok(prepared) +} +fn payloads( + store: &Store, + journal: &Dir, + record: &Journal, + frozen: Option<&Plan>, +) -> Result, String> { + let ownership = record + .ownership + .as_ref() + .ok_or("Missing persisted prepared identities")?; + let planned = frozen + .into_iter() + .flat_map(|plan| &plan.files) + .map(|file| (file.name.as_str(), file)) + .collect::>(); + let mut files = Vec::new(); + for (index, change) in record.changes.iter().enumerate() { + let name = format!("stage_{index}"); + let image = ownership::observe_image(store, Some(journal), std::ffi::OsStr::new(&name))?; + let previous = planned.get(name.as_str()).copied(); + match (&change.after, image) { + (Some(after), Some(image)) + if after.kind == super::super::ImageKind::File + && image.entry == *after + && ownership.prepared_identity(index) == Some(image.identity.as_str()) => + { + files.push(Payload { + journal: None, + name, + identity: image.identity, + generation: image.generation, + hash: image.entry.object, + bytes: image.entry.length, + }) + } + (Some(after), None) + if previous.is_some_and(|file| { + file.hash == after.object + && file.bytes == after.length + && ownership.prepared_identity(index) == Some(file.identity.as_str()) + }) => + { + files.push(previous.cloned().ok_or("Missing frozen prepared leaf")?) + } + (None, None) if previous.is_none() => {} + _ => return Err("Prepared cancellation payload lacks persisted ownership".into()), + } + } + Ok(files) +} diff --git a/crates/cyber-snapshot/src/copy/reclamation/paired.rs b/crates/cyber-snapshot/src/copy/reclamation/paired.rs index 4a0ab5d0a..6f30f199e 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/paired.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/paired.rs @@ -183,6 +183,7 @@ fn prepare( )?); let plan = Plan { version: 2, + intent_digest: None, receipt: settlement.receipt, files, }; diff --git a/docs/rewind.md b/docs/rewind.md index 83a44b8c0..8449d9884 100644 --- a/docs/rewind.md +++ b/docs/rewind.md @@ -186,6 +186,13 @@ interrupted cleanup resumes only through its durable plan. Both intents and look receipts remain usable afterward. Managed Git, preparation-construction, metadata/ CAS and legacy payload reclamation remain unfinished. +Cancelled pre-effect preparations can reclaim fully prepared regular target images +after retention when their recorded inode identities and committed intent proof +still match. Cleanup preserves the intent and cancellation records and permits +later receipt acknowledgement without reading or restoring source files. Older +cancellations lacking committed intent proof and incomplete constructions remain +protected; their payloads are not inferred from filenames or matching bytes. + Before file changes, restore durably publishes an intent and prepares private stages. Replacements retain displaced inodes; progress and completion are flushed independently. Admission and content/route identities are rechecked. A failure or diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 060377a7b..a0767293b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3340,3 +3340,75 @@ without unchanged-main attribution. All Cargo tests/checks/Clippy used CARGO_INCREMENTAL=0 and --locked. No unresolved critical/major code-review finding remains within this paired regular-payload scope; the broader acceptance obligations listed above remain open and neither issue is complete. + +## Cancelled prepared-target reclamation (2026-11-07, local) + +The third M6 payload path handles positively cancelled pre-effect preparations +with a published origin, fully persisted prepared target inode identities, a +non-admitted version-2 native phase and an empty complete checkout set. It removes +only regular private target images. Intent, cancellation, lifetime/phase and request +metadata remain; source routes/images are never consulted for cleanup. Missing +intent/ownership, partial constructors, legacy proof, managed resources and Git +are not promoted into this admission and remain required/protected classes. + +New cancellation review fingerprints bind the immutable prepared intent digest +from a metadata-only native lifetime port. The consuming writer rechecks it and +persists snapshot_preparation_intent_digest on the exact cancelled native activity. +A version-3 reclamation plan freezes that digest and every prepared target leaf's +original inode, current generation, BLAKE3 identity and length. The maintenance +writer matches the committed digest before unlinks, while namespace/Location/ +lifetime guards and intent/cancellation/phase metadata pins remain retained. A +historical cancellation without this committed digest keeps its existing lookup +and acknowledgement path and cannot gain cleanup authority from current files. +Public preparation receipt/HTTP/SDK shapes are unchanged. Version-1/2 plans omit +the new optional plan digest field; their serialization and IDs remain stable. +Borrowed owner evidence avoids inventing ordinary-restore checkout receipts for +preparations. The larger preparation carrier is boxed without serialization change. + +The actual CLI fixture holds a real SQLite IMMEDIATE transaction after observing +origin publication. It edits no row: writer contention prevents source admission +while the real worker writes all target images and persists prepared ownership. +The fixture observes that publication, kills the real service and verifies the +Store still positively records no source admission. Real reviewed cancellation +and explicit native acknowledgement then precede maintenance over the reopened +actual Store. The later source symlink/target, other bytes, transcript and inference +remain unchanged, and a new prompt still succeeds after cleanup. + +Regression checks cover stale review after changing native intent, young retention, +a live original lifetime, replacement with identical bytes but a different inode, +a historical missing committed digest, changed intent after cancellation, unknown +entries, a committed plan cut before deletion, synthetic progress of one planned +unlink, retry/accounting and preserved intent/cancellation records. The latter two +cuts are synthetic cleanup states; the preparation service kill itself is real. +Lookup and metadata acknowledgement are exercised again after cleanup/restart. + +Code-review skill inspection found that inode identity alone needed an intent +anchor in the committed cancellation pair. The native metadata port, version-3 +review fingerprint and writer-bound digest check address that finding; current +native bytes never manufacture historical proof. Initial fixture assertions also +incorrectly required an explicitly serialized false acknowledgement flag; the +production serializer omits false and the fixture now respects that default. +Clippy identified test complexity 21/15 and 16/15; fixture setup, restart lookup, +plan interruption and digest checks were split by responsibility, with no lint +allowances. The Rust cognitive-complexity check uses workspace Clippy's threshold +15; the cognitive-complexity skill has no Rust analyzer. Existing project snapshot +ports and the application Store adapter keep filesystem and database ownership +separate. Full M6, native platform/cut/lease acceptance and both issues remain open. + +Final local macOS arm64 checks: the complete 19-case serial CLI branch/recovery +invocation passed in 405.81 seconds, including prepared cancellation cleanup, +paired keep/rollback, original pre-effect cuts and terminal process-death recovery. +The focused digest-bound preparation case passed in 74.80 seconds before its final +test-helper refactor. All 15 shared App cleanup cases passed in 41.88 seconds and +all 46 runtime snapshot-worker tests passed in 4.85 seconds. HTTP passed all 80 +cases in 4.09 seconds. Two initial library name filters selected zero tests; the +correct snapshot_work::tests filter produced the 46-case result above. All-target +App/server/snapshot/CLI Clippy, format/diff, generated SDK consistency, strict +OpenSpec (117 passed) and spec lint (0 errors/25 warnings) passed. All Cargo tests, +checks and Clippy used CARGO_INCREMENTAL=0 and --locked. C sandbox deprecations +and linker unwind warnings were observed without unchanged-main attribution. +No full workspace/snapshot-library/SDK test run, separate native platform or CI +execution ran for this increment. No unresolved critical/major review finding +remains in this fully prepared regular-target scope; raw construction, managed/Git, +metadata/CAS and full acceptance remain required. No PR/push or parent completion +is claimed. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index d4f1c58e1..f63158185 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -850,3 +850,24 @@ Neither source bytes nor current source routes grant cleanup authority or need t be read. Both intents, settlement links, phase metadata and durable lookup receipts SHALL survive payload reclamation. Retention SHALL begin at durable paired settlement and be rechecked during deletion; unknown native owners continue to protect both. + +### Cancelled prepared-image cleanup + +An exact preparation cancellation with a published origin, positive non-admitted +version-2 phase, complete empty checkout set and matching native cancellation +metadata MAY reclaim fully prepared regular target images. Each leaf SHALL match +its persisted prepared inode identity and immutable target image; incomplete +construction without those identities SHALL remain protected. The version-3 plan +SHALL freeze the native intent digest and complete private inventory before +unlinking. Intent, cancellation, phase and durable request metadata SHALL remain. +Native acknowledgement and cleanup SHALL NOT fabricate completion or source +admission. Retention, references, reader/lifetime guards and independent unknown +fences SHALL use the same consuming-writer protocol as other cleanup classes. + +For newly cancelled fully prepared images, the read-only native lifetime port +returns the immutable intent digest. Review fingerprints bind it; consuming-writer +revalidation checks it again and persists `snapshot_preparation_intent_digest` in +the native cancellation event. Version-3 cleanup MUST match that exact committed +digest. Missing historical digest evidence cannot be inferred from current native +files and grants no cleanup authority; legacy lookup/metadata acknowledgement +retains its existing behavior. This adds no public receipt field or source replay. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md index 19ed3d8c4..2eeeea4ba 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/storage-events/spec.md @@ -51,3 +51,10 @@ - **AND** both exited lifetimes and complete inventories SHALL be checked before any unlink - **AND** interrupted deletion SHALL resume only from that durable plan without reading or replaying source - **AND** original version-1 restore plan serialization and IDs SHALL remain unchanged + +#### Scenario: Cancelled prepared target cleanup +- **GIVEN** a positive pre-effect cancellation binds fully prepared native target identities and an immutable intent digest +- **WHEN** its cancellation is durably committed and eligible private payloads are reclaimed +- **THEN** the native cancellation event SHALL retain the reviewed intent digest and a version-3 reclamation plan SHALL match it +- **AND** missing historical digest evidence or changed target inodes SHALL refuse cleanup without creating new acknowledgement authority +- **AND** cancellation, intent and request lookup metadata SHALL remain after payload deletion without source admission or source replay diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 25723a5cf..0d03ce1ee 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -541,3 +541,14 @@ Exercise the real partial-restore kill fixture for keep and rollback, both live worker locks, young retention, unknown entries and planned partial progress. Construction/preparation payloads, managed resources/Git, CAS/metadata reclamation and the complete native cut/platform matrix remain required; M6 stays open. + +M6 prepared-cancellation continuation: bind the immutable prepared intent into +review and the durable cancellation activity, and admit version-3 target cleanup +only with persisted prepared inode identities, exact native cancellation and a +positive non-admitted phase. Retain all intent/cancellation/phase/request metadata; +unknown/incomplete construction and historical missing digest evidence cannot be +promoted into cleanup proof. Exercise an actual service kill with prepared images +while SQLite writer contention prevents source admission, then verify changed +inodes/digests, historical missing proof and durable partial cleanup. Raw partial +construction, managed/Git resources, metadata/CAS and full platform/cut acceptance +remain required; all parent tasks stay open. From ee6ad80e3d738599b7ed4f69d2ba38c84e79b14a Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 12:13:17 -0400 Subject: [PATCH 45/87] Bind native Git handoffs to user index observations --- .../cyber-snapshot/src/copy/config_inputs.rs | 4 +- crates/cyber-snapshot/src/copy/git/tests.rs | 209 ++++++++++++++++++ crates/cyber-snapshot/src/copy/git_rules.rs | 2 +- crates/cyber-snapshot/src/git.rs | 2 + crates/cyber-snapshot/src/git/catalog.rs | 11 +- crates/cyber-snapshot/src/git/prepared.rs | 1 + crates/cyber-snapshot/src/git/user_index.rs | 97 ++++++++ .../implementation.md | 60 +++++ .../managed-recovery.md | 14 ++ .../specs/snapshots-checkpoints/spec.md | 14 ++ .../complete-daily-coding-surfaces/tasks.md | 11 + 11 files changed, 420 insertions(+), 5 deletions(-) create mode 100644 crates/cyber-snapshot/src/git/user_index.rs diff --git a/crates/cyber-snapshot/src/copy/config_inputs.rs b/crates/cyber-snapshot/src/copy/config_inputs.rs index e83d47650..dfbc7a240 100644 --- a/crates/cyber-snapshot/src/copy/config_inputs.rs +++ b/crates/cyber-snapshot/src/copy/config_inputs.rs @@ -108,7 +108,7 @@ impl ConfigInputs { } self.contexts.insert(path, state); } - if self.files.len() > 1024 || self.contexts.len() > 4 { + if self.files.len() > 1024 || self.contexts.len() > 8 { return Err("Combined Git policy inputs exceed limits".into()); } self.verify() @@ -121,7 +121,7 @@ impl ConfigInputs { } pub(crate) fn bind_contexts(&mut self, paths: Vec) -> storage::Result<()> { - if paths.len() > 4 { + if paths.len() > 8 { return Err("Git configuration directory context exceeds limit".into()); } for path in paths { diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index 0bac7d567..d7ca9178a 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -114,6 +114,180 @@ async fn changed_user_excludes_after_preparation_refuses_native_source_effects() assert!(store.recovery().unwrap().is_empty()); } +#[tokio::test] +async fn changed_user_index_after_preparation_refuses_native_source_effects() { + let mut repo = Repo::new(); + repo.snapshots = crate::GitSnapshots::new( + repo.data.clone(), + Arc::new(|_| serde_json::json!({"snapshots":{"max_file_bytes":4096}})), + ); + std::fs::write(repo.root.join("large"), vec![b'x'; 5120]).unwrap(); + let git = |args: &[&str]| { + assert!( + std::process::Command::new("git") + .arg("-C") + .arg(&repo.root) + .args(args) + .status() + .unwrap() + .success() + ); + }; + git(&["add", "--", "large"]); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + git(&["rm", "--cached", "--quiet", "--", "large"]); + let store = repo.store(&repo.root); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert_eq!( + std::fs::read(repo.root.join("large")).unwrap(), + vec![b'x'; 5120] + ); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn native_git_index_proof_rejects_creation_replacement_and_changed_split_dependency() { + for kind in ["creation", "replacement", "split"] { + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), "before").unwrap(); + if kind != "creation" { + repo.git(&["add", "--", "file"]); + } + if kind == "split" { + repo.git(&["update-index", "--split-index"]); + } + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + match kind { + "creation" => repo.git(&["add", "--", "file"]), + "replacement" => { + let index = repo.root.join(".git/index"); + let replacement = repo.root.join(".git/replacement"); + std::fs::write(&replacement, std::fs::read(&index).unwrap()).unwrap(); + std::fs::rename(replacement, index).unwrap(); + } + "split" => { + let shared = std::fs::read_dir(repo.root.join(".git")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("sharedindex.") + }) + .unwrap(); + std::fs::write(shared, "changed split index").unwrap(); + } + _ => unreachable!(), + } + assert!(prepared.ignore.config.verify().is_err(), "{kind}"); + let store = repo.store(&repo.root); + assert!(store.restore_git_worker(prepared, None).is_err(), "{kind}"); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); + } +} + +#[tokio::test] +async fn native_git_split_index_preparation_and_restore_preserve_user_index_metadata() { + use std::os::unix::fs::MetadataExt; + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), "before").unwrap(); + repo.git(&["add", "--", "file"]); + repo.git(&["update-index", "--split-index"]); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + let shared = std::fs::read_dir(repo.root.join(".git")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.file_name() + .unwrap() + .as_encoded_bytes() + .starts_with(b"sharedindex.") + }) + .unwrap(); + let paths = [repo.root.join(".git/index"), shared]; + let observations = || { + paths.each_ref().map(|path| { + let metadata = std::fs::metadata(path).unwrap(); + ( + metadata.ino(), + metadata.mtime(), + metadata.mtime_nsec(), + metadata.ctime(), + metadata.ctime_nsec(), + std::fs::read(path).unwrap(), + ) + }) + }; + let before = observations(); + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + assert_eq!(observations(), before); + repo.store(&repo.root) + .restore_git_worker(prepared, None) + .unwrap(); + assert_eq!(observations(), before); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"before"); +} + +#[tokio::test] +async fn oversized_user_index_keeps_canonical_capture_without_native_authority() { + use sha1::Digest; + let repo = Repo::new(); + std::fs::write(repo.root.join("file"), "canonical-readable").unwrap(); + repo.git(&["add", "--", "file"]); + let path = repo.root.join(".git/index"); + let mut bytes = std::fs::read(&path).unwrap(); + bytes.truncate(bytes.len() - 20); + bytes.extend_from_slice(b"TEST"); + bytes.extend_from_slice(&(1024_u32 * 1024).to_be_bytes()); + bytes.resize(bytes.len() + 1024 * 1024, 0); + let hash = sha1::Sha1::digest(&bytes); + bytes.extend_from_slice(&hash); + std::fs::write(&path, &bytes).unwrap(); + repo.git(&["ls-files", "--error-unmatch", "--", "file"]); + let snapshot = repo + .snapshots + .track(repo.root.to_str().unwrap()) + .await + .unwrap() + .unwrap(); + assert!( + snapshot + .skipped + .iter() + .any(|value| value.contains("native Git policy/raw-state proof unavailable")) + ); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + assert_eq!( + shadow + .blob(&snapshot.tree, "file") + .await + .unwrap() + .unwrap() + .1, + b"canonical-readable" + ); + assert_eq!(std::fs::read(path).unwrap(), bytes); +} + #[tokio::test] async fn changed_git_config_locator_after_preparation_refuses_native_source_effects() { let repo = Repo::new(); @@ -203,6 +377,17 @@ async fn native_git_config_proof_rejects_include_creation_changes_and_conditiona } impl Repo { + fn git(&self, args: &[&str]) { + assert!( + std::process::Command::new("git") + .arg("-C") + .arg(&self.root) + .args(args) + .status() + .unwrap() + .success() + ); + } fn new() -> Self { Self::with_format("sha1") } @@ -1708,9 +1893,11 @@ async fn sha256_linked_worktree_uses_its_common_repository_format_and_scoped_git let prepared = repo.prepare(&location, &target, &recorded, &recorded).await; assert_eq!(prepared.prefix, "nested"); assert_eq!(prepared.target.images["file"].bytes, b"before"); + let policy = prepared.ignore.config.clone(); repo.store(&location) .restore_git_worker(prepared, None) .unwrap(); + policy.verify().unwrap(); assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); std::fs::write(location.join("file"), "after").unwrap(); let paths = repo @@ -1740,6 +1927,28 @@ async fn sha256_linked_worktree_uses_its_common_repository_format_and_scoped_git assert!(store.restore_git_worker(prepared, None).is_err()); assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); assert_eq!(std::fs::read(linked.join(".git")).unwrap(), marker); + std::fs::remove_file(repo.root.join(".git/info/exclude")).unwrap(); + let expected = repo.track(&location).await; + let prepared = repo + .prepare(&location, &recorded, &expected, &expected) + .await; + let common_index = std::fs::read(repo.root.join(".git/index")).ok(); + assert!( + std::process::Command::new("git") + .arg("-C") + .arg(&linked) + .args(["add", "--", "nested/file"]) + .status() + .unwrap() + .success() + ); + assert!(prepared.ignore.config.verify().is_err()); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!( + std::fs::read(repo.root.join(".git/index")).ok(), + common_index + ); + assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); assert!( !repo .snapshots diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs index b8c977c61..d03537751 100644 --- a/crates/cyber-snapshot/src/copy/git_rules.rs +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -146,7 +146,7 @@ impl Rules { (left.source, &left.path, &left.hash).cmp(&(right.source, &right.path, &right.hash)) }); let bytes = serde_json::to_vec(&( - "git-policy-capture-v3", + "git-policy-capture-v4", &self.path, &self.inputs, bindings, diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 13bc5c7da..377a8e418 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -16,6 +16,8 @@ mod images; pub(crate) mod prepared; #[cfg(any(target_os = "macos", target_os = "linux"))] mod raw; +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod user_index; /// A shadow git directory bound to one worktree and scoped to one Location. #[derive(Debug, Clone)] diff --git a/crates/cyber-snapshot/src/git/catalog.rs b/crates/cyber-snapshot/src/git/catalog.rs index 224427a1e..0e2567a2f 100644 --- a/crates/cyber-snapshot/src/git/catalog.rs +++ b/crates/cyber-snapshot/src/git/catalog.rs @@ -32,8 +32,15 @@ impl Shadow { limit: usize, timeout: std::time::Duration, ) -> Result, String> { - let mut child = self - .command(args) + Self::bounded_command(self.command(args), limit, timeout).await + } + + pub(super) async fn bounded_command( + mut command: tokio::process::Command, + limit: usize, + timeout: std::time::Duration, + ) -> Result, String> { + let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index 539217eef..ac6970f1a 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -127,6 +127,7 @@ impl Shadow { &self.common_dir, settings, )?)?; + config.merge(self.user_index_inputs().await?)?; Ok(IgnoreInputs { config, info: self.git_dir.join("info/exclude"), diff --git a/crates/cyber-snapshot/src/git/user_index.rs b/crates/cyber-snapshot/src/git/user_index.rs new file mode 100644 index 000000000..651eb0167 --- /dev/null +++ b/crates/cyber-snapshot/src/git/user_index.rs @@ -0,0 +1,97 @@ +//! Read-only user-index observations for large-untracked eligibility handoff. +use super::Shadow; +use crate::copy::config_inputs::ConfigInputs; +use std::path::PathBuf; + +#[derive(PartialEq, Eq)] +struct Layout { + directory: PathBuf, + index: PathBuf, +} + +impl Shadow { + pub(super) async fn user_index_inputs(&self) -> Result { + // Observe discovery routing before Git resolves any metadata paths. + let marker = self.worktree.join(".git"); + let metadata = std::fs::symlink_metadata(&marker).map_err(|error| error.to_string())?; + if metadata.file_type().is_symlink() { + return Err("Unsupported user Git marker link".into()); + } + let mut proof = ConfigInputs::capture_files(if metadata.is_dir() { + vec![] + } else { + vec![marker.clone()] + })?; + let mut contexts = vec![self.worktree.clone()]; + if metadata.is_dir() { + contexts.push(marker); + } + proof.bind_contexts(contexts)?; + let layout = self.user_index_layout().await?; + let mut paths = vec![ + layout.index.clone(), + layout.directory.join("commondir"), + layout.directory.join("gitdir"), + ]; + paths.extend(shared_indexes(&layout.directory)?); + let mut index = ConfigInputs::capture_files(paths)?; + index.bind_contexts(vec![layout.directory.clone()])?; + proof.merge(index)?; + if self.user_index_layout().await? != layout { + return Err("User Git index layout changed during collection".into()); + } + proof.verify()?; + Ok(proof) + } + + async fn user_index_layout(&self) -> Result { + let mut command = self.command(&[ + "rev-parse", + "--absolute-git-dir", + "--path-format=absolute", + "--git-path", + "index", + ]); + command + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env("GIT_OPTIONAL_LOCKS", "0"); + let bytes = + Self::bounded_command(command, 32 * 1024, std::time::Duration::from_secs(10)).await?; + let text = std::str::from_utf8(&bytes).map_err(|_| "Non-UTF-8 user Git index path")?; + let lines: Vec<_> = text.lines().collect(); + if lines.len() != 2 { + return Err("Unsupported user Git index layout".into()); + } + let directory = PathBuf::from(lines[0]); + let index = PathBuf::from(lines[1]); + if !directory.is_absolute() || index != directory.join("index") { + return Err("Unsupported user Git index locator".into()); + } + Ok(Layout { directory, index }) + } +} + +// Resolving --shared-index-path through Git refreshes shared-index timestamps. +// Observe every bounded candidate instead: this includes the active dependency +// without reading the index through Git, and the index observation binds selection. +fn shared_indexes(directory: &std::path::Path) -> Result, String> { + let mut paths = Vec::new(); + for (count, entry) in std::fs::read_dir(directory) + .map_err(|error| error.to_string())? + .enumerate() + { + if count >= 4096 { + return Err("User Git index directory exceeds observation limit".into()); + } + let entry = entry.map_err(|error| error.to_string())?; + if entry + .file_name() + .as_encoded_bytes() + .starts_with(b"sharedindex.") + { + paths.push(entry.path()); + } + } + Ok(paths) +} diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index a0767293b..c8d20acbc 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3412,3 +3412,63 @@ execution ran for this increment. No unresolved critical/major review finding remains in this fully prepared regular-target scope; raw construction, managed/Git, metadata/CAS and full acceptance remain required. No PR/push or parent completion is claimed. + +### User Git index handoff evidence (2026-11-07) + +A new real-repository regression reproduced a native backend gap: after preparing +an expected checkpoint with a 5 KiB tracked file and a 4 KiB untracked limit, +removing that file from the user's index still allowed the native restore. The +initial regression failed its refusal assertion; this is reproduced branch +behavior, not an unchanged-main attribution. + +Native capture/preparation now binds the actual per-worktree index (including +absence), marker/admin routing and all bounded shared-index candidates into the +existing read-only metadata observations. User path discovery clears repository +redirection environment, disables optional locks, and uses the existing bounded +output/deadline/kill-and-await runner. The routing marker is observed before +resolution; resolved layout is checked again after index collection. The +synchronous native worker revalidates exact file observations and directory +identities without Git, index writes or source replays. A distinct v4 policy +fingerprint leaves older policy evidence unpromoted. The combined directory +context bound is eight so a linked worktree can retain its independent admin +identity alongside common/shadow/rule contexts. Existing 1 MiB per-file, 8 MiB +aggregate and 1024-file observation limits remain enforced. + +The first split-index implementation used `git rev-parse --shared-index-path`. +Its new regression failed because this nominally read-only command refreshes +shared-index mtime/ctime even with GIT_OPTIONAL_LOCKS=0. A direct temporary +repository experiment confirmed that metadata change. Collection now enumerates +at most 4096 admin entries and binds shared-index candidates directly, avoiding +that command and leaving selection bound by the exact index bytes. This is a +conservative superset: an unused candidate changing can refuse native preparation. +Oversized observations retain ordinary canonical capture with explicit native +proof unavailability. A valid oversized index with an optional extension exercises +that fallback and verifies that capture preserves user index bytes. + +Regression coverage includes post-preparation tracked-to-untracked changes, +absent-index creation, identical-byte inode replacement, shared-index dependency +changes, SHA-256 linked-worktree independent index changes, and successful split- +index preparation/restore preserving index/shared-index bytes, inode and mtime/ +ctime. The linked-worktree positive also revalidates the retained policy after +native source restoration. Code-review skill inspection checked routing before +resolution, bounded subprocess cleanup, native verification without subprocesses, +old fingerprint handling and canonical fallback. No critical/major finding remains +in this index-handoff increment; it does not prove size-decision generation or +complete user config dependencies. + +Local macOS arm64 validation: the full snapshot library run passed 126 tests with +four intentionally ignored subprocess fixtures in 44.16 seconds; the subsequently +added split-index metadata positive passed separately in 2.15 seconds. Integration +suites passed all 18 file-copy and 22 Git snapshot cases (one intentionally ignored +subprocess fixture). The Windows-only file_copy_platform binary selected zero +cases on this host and supplies no Windows evidence. Snapshot all-target Clippy, +format/diff, generated SDK consistency, strict OpenSpec (117 items) and spec lint +(0 errors/25 warnings) passed. Cargo invocations used CARGO_INCREMENTAL=0 and +--locked. No full workspace, CLI/TUI acceptance rerun, SDK execution, other native +platform or CI run was performed for this increment. + +Managed Git source admission stays disabled. Fresh generated large-untracked +classification, complete user config/filter compatibility, tracked-large-file +limits, runtime/checkout/recovery/cleanup and full crash/platform acceptance still +require implementation or proof. M2–M8, parent tasks and both issues remain open; +there is no push/PR/merge acceptance claim. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index f63158185..84d044788 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -735,6 +735,20 @@ publication SHALL use descriptor-relative replacement without following output links. This does not prove user-index or large-untracked eligibility changes; those and full configuration/filter compatibility still gate managed Git admission. +### User Git index handoff binding + +Native policy collection SHALL observe the user's actual per-worktree index, +including absence, and the active split-index dependency before and after Git +path resolution. Linked worktrees SHALL bind their own index and routing files, +rather than substituting the common repository index. The synchronous native +planner SHALL revalidate retained file generations, bytes and directory identities +without running Git or writing the user index. A missing, changed, oversized or +unsupported observation SHALL not become native restore authority. Ordinary +canonical capture SHALL remain available when native proof is unavailable. +This binding does not by itself prove the generated large-untracked decisions, +all user configuration dependencies, tracked-large-file compatibility or full Git +runtime acceptance; those obligations still gate managed Git source admission. + ### Typed recovery CLI dispatch `cyber sessions recovery ` SHALL list retained journals without inference. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index ad793a29b..c64014fc0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -371,3 +371,17 @@ - **THEN** cancellation SHALL require its exact reviewed fingerprint and stable request ID, with malformed/incomplete options refused before service startup - **AND** replies SHALL match the selected Session/activity/lifetime and any published origin; source-admitted, mismatched or unsupported review evidence SHALL NOT be presented as cancellation authority - **AND** `sessions recovery-preparation-request` SHALL look up the committed receipt without replay, keep acknowledgement reconciliation explicit, and preserve files, transcript, evidence and independent process/checkout uncertainty + + +#### Scenario: User Git index changes after native preparation +- **GIVEN** prepared native Git images bind the actual user worktree index, its absence or active split-index dependency, and repository routing +- **WHEN** an index is created, replaced or changed after preparation, including a linked worktree's independent index +- **THEN** the native planner SHALL refuse before journal/source effects by revalidating bounded retained observations without running Git or writing the user index +- **AND** a same-byte replacement SHALL NOT inherit the original index generation proof +- **AND** a distinct policy fingerprint frame SHALL NOT promote historical proof lacking user-index observations + +#### Scenario: User index exceeds native observation bounds +- **GIVEN** an otherwise usable Git repository has a user index exceeding the bounded native metadata reader +- **WHEN** a checkpoint is captured +- **THEN** ordinary canonical capture SHALL remain readable with explicit native-proof unavailability, and SHALL NOT acquire native restore authority or modify the user index +- **AND** index binding alone SHALL NOT prove large-untracked classification, complete user configuration dependencies or managed Git runtime acceptance diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 0d03ce1ee..75c813037 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -552,3 +552,14 @@ while SQLite writer contention prevents source admission, then verify changed inodes/digests, historical missing proof and durable partial cleanup. Raw partial construction, managed/Git resources, metadata/CAS and full platform/cut acceptance remain required; all parent tasks stay open. + +User-index handoff continuation: native policy now binds the actual per-worktree +index, absence, routing files and bounded shared-index candidates, with native +read-only revalidation and a distinct fingerprint frame. Regression coverage +includes tracked-to-untracked, creation, identical-byte inode replacement, split +dependency changes and SHA-256 linked-worktree index isolation. A positive verifies +split-index metadata/bytes remain unchanged through preparation and native restore; +oversized observations retain canonical capture without native authority. Size- +decision generation, complete user config/filter compatibility, tracked large +files, runtime/checkout/recovery/cleanup and full crash/platform gates remain open. +Managed Git source admission and all M2–M8/parent task acceptance remain unchanged. From b27f7e8420f8c1983859023f1f89de910b6a1aeb Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 12:22:39 -0400 Subject: [PATCH 46/87] Revalidate generated Git size exclusions before native effects --- .../src/copy/generated_rules.rs | 14 +- crates/cyber-snapshot/src/copy/git/tests.rs | 143 ++++++++++++++++++ crates/cyber-snapshot/src/copy/git_rules.rs | 26 +++- crates/cyber-snapshot/src/git/prepared.rs | 10 +- .../implementation.md | 62 ++++++++ .../managed-recovery.md | 17 +++ .../specs/snapshots-checkpoints/spec.md | 18 +++ .../complete-daily-coding-surfaces/tasks.md | 10 ++ 8 files changed, 291 insertions(+), 9 deletions(-) diff --git a/crates/cyber-snapshot/src/copy/generated_rules.rs b/crates/cyber-snapshot/src/copy/generated_rules.rs index 818bd084f..97d6197fd 100644 --- a/crates/cyber-snapshot/src/copy/generated_rules.rs +++ b/crates/cyber-snapshot/src/copy/generated_rules.rs @@ -91,11 +91,16 @@ fn publish(info: &Dir, name: &str, bytes: &[u8]) -> Result<()> { storage::atomic(info, name, bytes) } +pub(crate) struct Observed { + pub inputs: ConfigInputs, + pub size_excluded: Vec, +} + pub(crate) fn observe( git_dir: &Path, common: &Path, settings: &CaptureSettings, -) -> Result { +) -> Result { let root = Dir::open_ambient_dir(git_dir, cap_std::ambient_authority()).map_err(storage::fail)?; let info = storage::existing_child(&root, "info", false)?; @@ -123,7 +128,10 @@ pub(crate) fn observe( return Err("Generated Git ignore recipe or output changed".into()); } output.merge(source)?; - Ok(output) + Ok(Observed { + inputs: output, + size_excluded: recipe.skipped, + }) } #[cfg(test)] @@ -165,7 +173,7 @@ mod tests { .unwrap(); } fn proof(&self) -> Result { - observe(&self.shadow, &self.common, &self.settings) + observe(&self.shadow, &self.common, &self.settings).map(|observed| observed.inputs) } } diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index d7ca9178a..7c13c5456 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -114,6 +114,135 @@ async fn changed_user_excludes_after_preparation_refuses_native_source_effects() assert!(store.recovery().unwrap().is_empty()); } +#[tokio::test] +async fn shrinking_size_excluded_file_after_preparation_refuses_native_restore() { + let mut repo = Repo::new(); + repo.snapshots = crate::GitSnapshots::new( + repo.data.clone(), + Arc::new(|_| serde_json::json!({"snapshots":{"max_file_bytes":4096}})), + ); + std::fs::write(repo.root.join("large"), vec![b'x'; 5120]).unwrap(); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + std::fs::write(repo.root.join("large"), "now eligible").unwrap(); + let store = repo.store(&repo.root); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert_eq!( + std::fs::read(repo.root.join("large")).unwrap(), + b"now eligible" + ); + assert!(store.recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn native_git_size_exclusions_reject_late_type_and_threshold_changes() { + for kind in [ + "threshold", + "deleted", + "file-link", + "parent-link", + "directory", + "fifo", + ] { + let (repo, target, recorded) = Repo::size_fixture().await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + let inventory = store.git_inventory(&prepared).unwrap(); + mutate_size_exclusion(&repo, kind); + assert!(inventory.verify(&store).is_err(), "{kind}"); + assert!(store.preview_git(&prepared).is_err(), "{kind}"); + assert!(store.restore_git_worker(prepared, None).is_err(), "{kind}"); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert_eq!( + std::fs::read(repo._temp.path().join("outside/large[1]")).unwrap(), + vec![b'z'; 8192] + ); + assert!(store.recovery().unwrap().is_empty()); + } +} + +fn mutate_size_exclusion(repo: &Repo, kind: &str) { + let path = repo.root.join("nested/large[1]"); + let outside = repo._temp.path().join("outside"); + std::fs::create_dir(&outside).unwrap(); + std::fs::write(outside.join("large[1]"), vec![b'z'; 8192]).unwrap(); + match kind { + "threshold" => std::fs::write(path, vec![b'x'; 4096]).unwrap(), + "parent-link" => { + std::fs::rename( + repo.root.join("nested"), + repo._temp.path().join("old-nested"), + ) + .unwrap(); + std::os::unix::fs::symlink(outside, repo.root.join("nested")).unwrap(); + } + kind => { + std::fs::remove_file(&path).unwrap(); + match kind { + "deleted" => {} + "file-link" => std::os::unix::fs::symlink(outside.join("large[1]"), path).unwrap(), + "directory" => std::fs::create_dir(path).unwrap(), + "fifo" => assert!( + std::process::Command::new("mkfifo") + .arg(path) + .status() + .unwrap() + .success() + ), + _ => unreachable!(), + } + } + } +} + +#[tokio::test] +async fn native_git_size_exclusions_allow_still_oversized_edits_without_source_replay() { + let (repo, target, recorded) = Repo::size_fixture().await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let store = repo.store(&repo.root); + let inventory = store.git_inventory(&prepared).unwrap(); + std::fs::write(repo.root.join("nested/large[1]"), vec![b'y'; 8192]).unwrap(); + inventory.verify(&store).unwrap(); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"before"); + assert_eq!( + std::fs::read(repo.root.join("nested/large[1]")).unwrap(), + vec![b'y'; 8192] + ); +} + +#[tokio::test] +async fn fresh_git_capture_includes_newly_eligible_file_and_preserves_historical_omission() { + let (repo, target, recorded) = Repo::size_fixture().await; + std::fs::write(repo.root.join("nested/large[1]"), "now eligible").unwrap(); + let expected = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &expected) + .await; + assert_eq!( + prepared.expected.images["nested/large[1]"].bytes, + b"now eligible" + ); + assert!(prepared.ignore.size_excluded.is_empty()); + let store = repo.store(&repo.root); + assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"before"); + assert_eq!( + std::fs::read(repo.root.join("nested/large[1]")).unwrap(), + b"now eligible" + ); +} + #[tokio::test] async fn changed_user_index_after_preparation_refuses_native_source_effects() { let mut repo = Repo::new(); @@ -377,6 +506,20 @@ async fn native_git_config_proof_rejects_include_creation_changes_and_conditiona } impl Repo { + async fn size_fixture() -> (Self, String, String) { + let mut repo = Self::new(); + repo.snapshots = crate::GitSnapshots::new( + repo.data.clone(), + Arc::new(|_| serde_json::json!({"snapshots":{"max_file_bytes":4096}})), + ); + std::fs::create_dir(repo.root.join("nested")).unwrap(); + std::fs::write(repo.root.join("nested/large[1]"), vec![b'x'; 5120]).unwrap(); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + (repo, target, recorded) + } fn git(&self, args: &[&str]) { assert!( std::process::Command::new("git") diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs index d03537751..4b142b93a 100644 --- a/crates/cyber-snapshot/src/copy/git_rules.rs +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -40,6 +40,7 @@ impl Rules { bytes: 0, inputs: inputs.clone(), }; + rules.verify_sizes()?; let mut builder = rules.builder()?; for path in inputs.global.iter().chain(std::iter::once(&inputs.info)) { if let Some(bytes) = rules.bind(path, false)? { @@ -146,7 +147,7 @@ impl Rules { (left.source, &left.path, &left.hash).cmp(&(right.source, &right.path, &right.hash)) }); let bytes = serde_json::to_vec(&( - "git-policy-capture-v4", + "git-policy-capture-v5", &self.path, &self.inputs, bindings, @@ -155,7 +156,30 @@ impl Rules { .map_err(failed)?; Ok(blake3::hash(&bytes).to_hex().to_string()) } + fn verify_sizes(&self) -> Result<(), RestoreError> { + for path in &self.inputs.size_excluded { + super::tree::safe_path(path).map_err(RestoreError::Failed)?; + let path = Path::new(path); + let mut parent = self.root.try_clone().map_err(failed)?; + for part in path.parent().ok_or(RestoreError::Stale)?.components() { + parent = parent + .open_dir_nofollow(part.as_os_str()) + .map_err(|_| RestoreError::Stale)?; + } + let metadata = parent + .symlink_metadata(path.file_name().ok_or(RestoreError::Stale)?) + .map_err(|_| RestoreError::Stale)?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() <= self.inputs.settings.max_untracked + { + return Err(RestoreError::Stale); + } + } + Ok(()) + } pub(super) fn verify(&self) -> Result<(), RestoreError> { + self.verify_sizes()?; self.inputs .config .verify() diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index ac6970f1a..56fd1af7a 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -72,6 +72,7 @@ pub(crate) struct IgnoreInputs { pub case_insensitive: bool, pub private_roots: Vec, pub settings: CaptureSettings, + pub size_excluded: Vec, } pub(crate) struct PreparedRestore { @@ -122,11 +123,9 @@ impl Shadow { .map_or("false", |value| value.unwrap_or("true")); let ignore_case = self.config_boolean("core.ignorecase", raw).await?; self.verify_config_inputs(&config, &origins).await?; - config.merge(crate::copy::generated_rules::observe( - &self.git_dir, - &self.common_dir, - settings, - )?)?; + let generated = + crate::copy::generated_rules::observe(&self.git_dir, &self.common_dir, settings)?; + config.merge(generated.inputs)?; config.merge(self.user_index_inputs().await?)?; Ok(IgnoreInputs { config, @@ -135,6 +134,7 @@ impl Shadow { case_insensitive: ignore_case, private_roots: self.private_roots.clone(), settings: settings.clone(), + size_excluded: generated.size_excluded, }) } diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index c8d20acbc..4bcd1db70 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3472,3 +3472,65 @@ classification, complete user config/filter compatibility, tracked-large-file limits, runtime/checkout/recovery/cleanup and full crash/platform acceptance still require implementation or proof. M2–M8, parent tasks and both issues remain open; there is no push/PR/merge acceptance claim. + +### Generated size-exclusion revalidation (2026-11-07) + +The new shrink-after-preparation regression failed before this change: a regular +untracked file omitted above 4 KiB could shrink to a small eligible file while the +native planner continued using its old literal ignore pattern and restored another +source path. No unchanged-main attribution is made for this reproduced branch +behavior. Generated-rule observation now returns the size-excluded names decoded +from the exact recipe whose bytes/output/source proof is verified. IgnoreInputs +retains those names separately from ordinary ignore patterns. A v5 policy frame +includes the new inputs rather than upgrading older checkpoint evidence. + +Rules checks each size-excluded name at open and final capture/planning revalidation. +Parents are opened component by component without following links; leaf metadata +must still describe a regular file whose length exceeds the configured untracked +threshold. Omitted file contents are never opened or hashed by these checks. The +native planner rejects missing/smaller files, links, linked parents, directories +and FIFOs before intent/source admission. Bytes can change while the file remains +oversized; normal restoration of independent eligible paths preserves that file. +A fresh capture regenerates rules and includes a newly eligible raw image, while +historical target/recorded omissions continue to prevent fabricated deletion or +restoration. This keeps current expected validation distinct from historical +mutation restrictions. + +Real-repository tests cover the reproduced shrink, exact-threshold change, +deletion, leaf/parent links to a preserved external oversized target, directory and +FIFO substitution after an initial successful inventory. Each late inventory, +preview and restore refuses and leaves source/history intact. Positives cover +nested literal bracket names, a still-oversized owner edit and a fresh expected +capture retaining the newly eligible image without rewriting it during historical +restore. Code-review skill inspection checked recipe/input byte binding, bounded +name/path handling, descriptor traversal, metadata-only omitted-file observation, +fingerprint compatibility and positives rather than universal refusal. No +critical/major finding remains in this size-revalidation scope. + +The full local macOS arm64 snapshot library run passed 131 tests with four +intentionally ignored subprocess fixtures in 48.57 seconds. Strict OpenSpec passed +117 items, spec lint reported 0 errors/25 warnings and SDK generation consistency +passed. Complete check results for this increment follow below. + +This does not yet bind original size-classification generation to the exact user +index/config inputs; collection-time state equality alone cannot manufacture that +provenance. A temporary repository experiment also confirmed that a private admin +copy of index/shared-index files can isolate Git's split-index timestamp refresh +(the original shared file remained unchanged, the copy changed), but complete +configuration/conditional-include fidelity is required before adopting such a +classifier. That experiment is not implementation or acceptance. Tracked-large-file +limits, full configuration/filter compatibility, managed Git runtime/checkout/ +recovery/cleanup and full crash/platform evidence still gate source admission. +M2–M8, parent tasks and both issues remain open. + +Final checks for this increment: snapshot all-target Clippy with -D warnings, +format and diff checks passed. Public integration passed 18 file-copy cases in +1.09 seconds and 22 Git snapshot cases in 13.92 seconds (one deliberately ignored +subprocess fixture). The Windows-only platform binary selected zero cases on this +host; it supplies no Windows evidence. All Cargo tests/Clippy used +CARGO_INCREMENTAL=0 and --locked. No full workspace, CLI/TUI acceptance rerun, +SDK execution, Linux/Windows/macOS Intel or source-bound CI run was performed. +The GitHub refresh still reports #93 and #92 open and origin/main remains +`db9e171e`, already contained in this branch. PR #115 concerns model fallback and +is outside this recovery increment; it was left untouched. No goal PR/push/merge +or parent completion is claimed. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 84d044788..b2871cb02 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -749,6 +749,23 @@ This binding does not by itself prove the generated large-untracked decisions, all user configuration dependencies, tracked-large-file compatibility or full Git runtime acceptance; those obligations still gate managed Git source admission. +### Revalidation of generated size exclusions + +Native policy inputs SHALL retain the size-excluded names decoded from the same +verified generated-rule recipe as its output. Capture and the synchronous native +planner SHALL verify that each such name is still a regular file exceeding the +configured untracked size threshold. These checks SHALL traverse parents without +following links and inspect leaf metadata without reading omitted contents. A +missing file, smaller file, link or unsupported type SHALL refuse the stale +native plan. Changing bytes while remaining oversized SHALL not invent a conflict +or source mutation for an omitted file. These names and the threshold SHALL be +part of a distinct policy fingerprint frame. A new capture SHALL regenerate size +exclusions so newly eligible files participate in expected-state validation, while +historical omissions continue to protect restoration. This does not prove that a +recipe's classification was derived from the same user index/configuration during +its original generation; that separate capture-provenance requirement and full Git +runtime acceptance remain necessary before managed Git admission. + ### Typed recovery CLI dispatch `cyber sessions recovery ` SHALL list retained journals without inference. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index c64014fc0..241fb6b21 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -385,3 +385,21 @@ - **WHEN** a checkpoint is captured - **THEN** ordinary canonical capture SHALL remain readable with explicit native-proof unavailability, and SHALL NOT acquire native restore authority or modify the user index - **AND** index binding alone SHALL NOT prove large-untracked classification, complete user configuration dependencies or managed Git runtime acceptance + + +#### Scenario: Size-excluded Git file becomes eligible or changes type +- **GIVEN** capture-bound generated rules omit a regular untracked file because it exceeds the configured size threshold +- **WHEN** it shrinks to or below the threshold, disappears, becomes a link or unsupported type, or gains a linked parent before native planning completes +- **THEN** capture/native inventory SHALL refuse the stale size exclusion using no-follow metadata checks without opening omitted contents or following links +- **AND** no journal/source effect SHALL be admitted by that stale plan + +#### Scenario: Omitted Git file remains oversized +- **GIVEN** a verified size exclusion still refers to a regular file exceeding the threshold +- **WHEN** its owner changes its contents while it remains oversized +- **THEN** native restore SHALL preserve the omitted file and may restore independently changed eligible paths after normal policy/namespace checks + +#### Scenario: Fresh expected capture after a size exclusion expires +- **WHEN** a formerly size-excluded file shrinks and a new expected checkpoint is captured +- **THEN** the current capture SHALL include its exact raw image and remove the expired generated size exclusion +- **AND** target/recorded historical omissions SHALL continue to protect that file from fabricated deletion/restoration +- **AND** this size revalidation SHALL NOT imply original classification provenance, complete user config dependencies or managed Git runtime acceptance diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 75c813037..fd9bcf482 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -563,3 +563,13 @@ oversized observations retain canonical capture without native authority. Size- decision generation, complete user config/filter compatibility, tracked large files, runtime/checkout/recovery/cleanup and full crash/platform gates remain open. Managed Git source admission and all M2–M8/parent task acceptance remain unchanged. + +Generated size-exclusion continuation: retain size-excluded names from the same +verified recipe as generated output, include them in a distinct policy frame, and +recheck regular-file type and threshold through no-follow metadata at capture and +native planning revalidation. The reproduced shrink-after-preparation gap now +refuses before journal/source effects. Threshold/deletion/link/parent-link/FIFO +cases and still-oversized/fresh-capture historical-exclusion positives are covered. +Original classification provenance at generation, full configuration/filter/limit +compatibility and runtime/checkout/recovery/cleanup/platform acceptance remain +required; managed Git source admission and parent task acceptance stay disabled. From e5ba38d4f3e4ce4f08faf69a6dff5366145a20b3 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 13:34:55 -0400 Subject: [PATCH 47/87] Bind Git size classification to observed user inputs --- crates/cyber-snapshot/Cargo.toml | 1 + .../cyber-snapshot/src/copy/config_inputs.rs | 41 ++ .../src/copy/generated_rules.rs | 56 ++- crates/cyber-snapshot/src/copy/git/tests.rs | 275 +++++++++++- crates/cyber-snapshot/src/copy/git_rules.rs | 137 +++++- crates/cyber-snapshot/src/git.rs | 71 ++-- crates/cyber-snapshot/src/git/config.rs | 151 ++++++- crates/cyber-snapshot/src/git/eligibility.rs | 395 ++++++++++++++++++ crates/cyber-snapshot/src/git/prepared.rs | 13 +- crates/cyber-snapshot/src/git/user_index.rs | 18 +- .../implementation.md | 112 +++++ .../managed-recovery.md | 32 ++ .../specs/snapshots-checkpoints/spec.md | 29 ++ .../complete-daily-coding-surfaces/tasks.md | 14 + 14 files changed, 1266 insertions(+), 79 deletions(-) create mode 100644 crates/cyber-snapshot/src/git/eligibility.rs diff --git a/crates/cyber-snapshot/Cargo.toml b/crates/cyber-snapshot/Cargo.toml index 9b3c6d531..6c824df90 100644 --- a/crates/cyber-snapshot/Cargo.toml +++ b/crates/cyber-snapshot/Cargo.toml @@ -27,6 +27,7 @@ ulid.workspace = true [target.'cfg(unix)'.dependencies] libc.workspace = true +tempfile.workspace = true rustix = { version = "=1.1.5", default-features = false, features = ["fs", "std"] } [dev-dependencies] diff --git a/crates/cyber-snapshot/src/copy/config_inputs.rs b/crates/cyber-snapshot/src/copy/config_inputs.rs index dfbc7a240..5c01a1507 100644 --- a/crates/cyber-snapshot/src/copy/config_inputs.rs +++ b/crates/cyber-snapshot/src/copy/config_inputs.rs @@ -25,6 +25,7 @@ struct FileState { canonical: PathBuf, parent_identity: String, generation: String, + identity: String, hash: String, } @@ -77,6 +78,32 @@ impl ConfigInputs { Ok(bytes) } + pub(crate) fn optional_bytes(&self, path: &Path) -> storage::Result>> { + let bytes = self.bytes(path)?; + Ok(self.files.get(path).and_then(Option::as_ref).map(|_| bytes)) + } + + /// Only disposable classifier copies use this check. Git refreshes shared + /// index timestamps, but their bytes, inode, attributes and route must match. + /// Original user inputs always use strict verify(), including timestamps. + pub(crate) fn verify_private_index_copies(&self) -> storage::Result<()> { + let mut current = Self::capture_files(self.files.keys().cloned().collect())?; + for (path, state) in &mut current.files { + if path + .file_name() + .is_some_and(|name| name.as_encoded_bytes().starts_with(b"sharedindex.")) + && let (Some(current), Some(Some(expected))) = + (state.as_mut(), self.files.get(path)) + { + current.generation = expected.generation.clone(); + } + } + if current != *self { + return Err("Private Git classifier inputs changed".into()); + } + Ok(()) + } + pub(crate) fn fingerprint(&self) -> storage::Result { Ok(storage::digest( &serde_json::to_vec(self).map_err(storage::fail)?, @@ -194,6 +221,19 @@ fn read(path: &Path) -> storage::Result<(Option, Vec)> { return Err("Unsupported or over-limit Git configuration input".into()); } let generation = ownership::generation(&file)?; + use std::os::unix::fs::MetadataExt; + let identity = storage::digest( + &serde_json::to_vec(&( + metadata.dev(), + metadata.ino(), + metadata.mode(), + metadata.uid(), + metadata.gid(), + metadata.nlink(), + metadata.len(), + )) + .map_err(storage::fail)?, + ); let canonical = path.canonicalize().map_err(storage::fail)?; let parent = cap_std::fs::Dir::open_ambient_dir( canonical.parent().ok_or("Missing config parent")?, @@ -225,6 +265,7 @@ fn read(path: &Path) -> storage::Result<(Option, Vec)> { canonical, parent_identity, generation, + identity, hash: storage::digest(&bytes), }), bytes, diff --git a/crates/cyber-snapshot/src/copy/generated_rules.rs b/crates/cyber-snapshot/src/copy/generated_rules.rs index 97d6197fd..1b30303af 100644 --- a/crates/cyber-snapshot/src/copy/generated_rules.rs +++ b/crates/cyber-snapshot/src/copy/generated_rules.rs @@ -15,6 +15,8 @@ struct Recipe { settings: CaptureSettings, skipped: Vec, output: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + classification: Option, } fn source(common: &Path) -> Result<(ConfigInputs, Vec)> { @@ -59,15 +61,20 @@ pub(crate) fn write( common: &Path, settings: &CaptureSettings, skipped: &[String], + classification: Option<(&str, &ConfigInputs)>, ) -> Result<()> { + if let Some((_, inputs)) = classification { + inputs.verify()?; + } let (proof, original) = source(common)?; let bytes = render(original, settings, skipped)?; let recipe = serde_json::to_vec(&Recipe { - version: 1, + version: if classification.is_some() { 2 } else { 1 }, source: proof.fingerprint()?, settings: settings.clone(), skipped: skipped.to_vec(), output: storage::digest(&bytes), + classification: classification.map(|(digest, _)| digest.to_owned()), }) .map_err(storage::fail)?; if recipe.len() as u64 > LIMIT { @@ -79,6 +86,9 @@ pub(crate) fn write( proof.verify()?; publish(&info, "exclude", &bytes)?; publish(&info, RECIPE, &recipe)?; + if let Some((_, inputs)) = classification { + inputs.verify()?; + } proof.verify() } @@ -94,6 +104,7 @@ fn publish(info: &Dir, name: &str, bytes: &[u8]) -> Result<()> { pub(crate) struct Observed { pub inputs: ConfigInputs, pub size_excluded: Vec, + pub classification: Option, } pub(crate) fn observe( @@ -113,7 +124,13 @@ pub(crate) fn observe( let encoded = storage::read(&info, RECIPE, LIMIT)?; let recipe: Recipe = serde_json::from_slice(&encoded).map_err(storage::fail)?; let (source, original) = source(common)?; - if recipe.version != 1 + if !matches!( + (recipe.version, recipe.classification.as_deref()), + (1, None) | (2, Some(_)) + ) || recipe + .classification + .as_deref() + .is_some_and(|digest| !storage::valid_digest(digest)) || recipe.settings != *settings || recipe.source != source.fingerprint()? { @@ -131,6 +148,7 @@ pub(crate) fn observe( Ok(Observed { inputs: output, size_excluded: recipe.skipped, + classification: recipe.classification, }) } @@ -169,6 +187,7 @@ mod tests { &self.common, &self.settings, &["big[1]".into()], + None, ) .unwrap(); } @@ -244,7 +263,16 @@ mod tests { } std::fs::rename(fixture.shadow.join("info"), fixture.shadow.join("old-info")).unwrap(); symlink(fixture.common.join("info"), fixture.shadow.join("info")).unwrap(); - assert!(write(&fixture.shadow, &fixture.common, &fixture.settings, &[]).is_err()); + assert!( + write( + &fixture.shadow, + &fixture.common, + &fixture.settings, + &[], + None + ) + .is_err() + ); assert!(fixture.proof().is_err()); assert!(!fixture.common.join("info").join(RECIPE).exists()); } @@ -262,7 +290,16 @@ mod tests { ); fixture.proof().unwrap().verify().unwrap(); std::fs::write(&path, vec![0; LIMIT as usize + 1]).unwrap(); - assert!(write(&fixture.shadow, &fixture.common, &fixture.settings, &[]).is_err()); + assert!( + write( + &fixture.shadow, + &fixture.common, + &fixture.settings, + &[], + None + ) + .is_err() + ); std::fs::remove_file(&path).unwrap(); assert!( std::process::Command::new("mkfifo") @@ -271,6 +308,15 @@ mod tests { .unwrap() .success() ); - assert!(write(&fixture.shadow, &fixture.common, &fixture.settings, &[]).is_err()); + assert!( + write( + &fixture.shadow, + &fixture.common, + &fixture.settings, + &[], + None + ) + .is_err() + ); } } diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index 7c13c5456..ba852d6b4 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -114,6 +114,258 @@ async fn changed_user_excludes_after_preparation_refuses_native_source_effects() assert!(store.recovery().unwrap().is_empty()); } +#[test] +fn git_config_command_override_cannot_redirect_snapshot_configuration() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("external-config"); + let original = b"[core]\nexcludesfile=never-read\n"; + std::fs::write(&path, original).unwrap(); + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "copy::git::tests::git_config_override_child", + ]) + .env("GIT_CONFIG", &path) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + assert_eq!(std::fs::read(path).unwrap(), original); +} + +#[tokio::test] +#[ignore = "subprocess fixture with isolated Git config command override"] +async fn git_config_override_child() { + let (repo, target, recorded) = Repo::size_fixture().await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + assert!(prepared.expected_policy.is_some()); + assert_eq!( + repo.store(&repo.root) + .restore_git_worker(prepared, None) + .unwrap(), + ["file"] + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"before"); +} + +#[tokio::test] +async fn frozen_classification_resolves_user_gitdir_and_branch_includes_in_original_context() { + let mut repo = Repo::new(); + repo.snapshots = crate::GitSnapshots::new( + repo.data.clone(), + Arc::new(|_| serde_json::json!({"snapshots":{"max_file_bytes":4096}})), + ); + let global = repo._temp.path().join("conditional-ignore"); + std::fs::write(&global, "ignored\n").unwrap(); + let include = repo._temp.path().join("conditional-config"); + std::fs::write( + &include, + format!("[core]\nexcludesfile={}\n", global.display()), + ) + .unwrap(); + repo.git(&[ + "config", + &format!("includeIf.gitdir:{}.path", repo.root.join(".git").display()), + include.to_str().unwrap(), + ]); + let head = std::fs::read_to_string(repo.root.join(".git/HEAD")).unwrap(); + let branch = head.trim().strip_prefix("ref: refs/heads/").unwrap(); + let branch_config = repo._temp.path().join("branch-config"); + std::fs::write( + &branch_config, + "[custom]\nflag\nvalue=\"quoted \\\"value\\\"\"\n", + ) + .unwrap(); + repo.git(&[ + "config", + &format!("includeIf.onbranch:{branch}.path"), + branch_config.to_str().unwrap(), + ]); + repo.git(&["config", "--get", "core.excludesfile"]); + repo.git(&["config", "--bool", "--get", "custom.flag"]); + std::fs::create_dir(repo.root.join("nested")).unwrap(); + std::fs::write(repo.root.join("nested/large[1]"), vec![b'x'; 5120]).unwrap(); + std::fs::write(repo.root.join("ignored"), vec![b'i'; 8192]).unwrap(); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + assert_eq!(prepared.ignore.size_excluded, ["nested/large[1]"]); + assert!(prepared.expected.images.contains_key("ignored")); + assert_eq!( + repo.store(&repo.root) + .restore_git_worker(prepared, None) + .unwrap(), + ["file"] + ); + assert_eq!( + std::fs::read(repo.root.join("ignored")).unwrap(), + vec![b'i'; 8192] + ); + assert!(!repo.root.join(".git/index").exists()); +} + +#[test] +fn classification_mutation_keeps_canonical_capture_without_native_proof() { + let temporary = tempfile::tempdir().unwrap(); + let bin = temporary.path().join("bin"); + std::fs::create_dir(&bin).unwrap(); + let wrapper = bin.join("git"); + std::fs::write(&wrapper, r#"#!/bin/sh +listing=0 +for argument in "$@"; do + if [ "$argument" = ls-files ]; then listing=1; fi +done +case "$GIT_DIR" in + */.eligibility-*) + if [ "$listing" = 1 ] && [ ! -f "$CYBER_CLASSIFY_CUT" ]; then + "$CYBER_REAL_GIT" "$@" > "$GIT_DIR/listing-output" || exit "$?" + touch "$CYBER_CLASSIFY_CUT" + case "$CYBER_CLASSIFY_MUTATION" in + source-index) env -u GIT_DIR -u GIT_WORK_TREE -u GIT_INDEX_FILE "$CYBER_REAL_GIT" -C "$GIT_WORK_TREE" add -- 'nested/large[1]' || exit "$?" ;; + private-config) printf '\n# changed private config\n' >> "$GIT_DIR/config" ;; + private-route) printf 'foreign-route\n' > "$GIT_DIR/commondir" ;; + esac + cat "$GIT_DIR/listing-output" + result=$? + rm "$GIT_DIR/listing-output" + exit "$result" + fi + ;; +esac +exec "$CYBER_REAL_GIT" "$@" +"#).unwrap(); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o700)).unwrap(); + let real = std::process::Command::new("which") + .arg("git") + .output() + .unwrap(); + assert!(real.status.success()); + let real = std::str::from_utf8(&real.stdout).unwrap().trim(); + for kind in ["source-index", "private-config", "private-route"] { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "copy::git::tests::classification_index_change_child", + ]) + .env("CYBER_REAL_GIT", real) + .env("CYBER_CLASSIFY_MUTATION", kind) + .env( + "CYBER_CLASSIFY_CUT", + temporary.path().join(format!("cut-{kind}")), + ) + .env( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap()), + ) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + } +} + +#[tokio::test] +#[ignore = "subprocess fixture with isolated Git classification wrapper"] +async fn classification_index_change_child() { + let repo = Repo::new(); + std::fs::create_dir(repo.root.join("nested")).unwrap(); + std::fs::write(repo.root.join("nested/large[1]"), vec![b'x'; 5120]).unwrap(); + std::fs::write(repo.root.join("file"), "before").unwrap(); + let mut repo = repo; + repo.snapshots = crate::GitSnapshots::new( + repo.data.clone(), + Arc::new(|_| serde_json::json!({"snapshots":{"max_file_bytes":4096}})), + ); + let snapshot = repo + .snapshots + .track(repo.root.to_str().unwrap()) + .await + .unwrap() + .unwrap(); + assert!(std::path::Path::new(&std::env::var_os("CYBER_CLASSIFY_CUT").unwrap()).exists()); + assert!( + snapshot + .skipped + .iter() + .any(|reason| reason.contains("native Git policy/raw-state proof unavailable")) + ); + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + let blob = shadow + .blob(&snapshot.tree, "nested/large[1]") + .await + .unwrap(); + if std::env::var("CYBER_CLASSIFY_MUTATION").unwrap() == "source-index" { + assert_eq!(blob.unwrap().1, vec![b'x'; 5120]); + } else { + assert!(blob.is_none()); + } + assert_eq!( + std::fs::read(repo.root.join("nested/large[1]")).unwrap(), + vec![b'x'; 5120] + ); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"before"); + assert!(repo.store(&repo.root).recovery().unwrap().is_empty()); +} + +#[tokio::test] +async fn empty_directories_do_not_change_capture_or_classification_identity() { + let (repo, _, recorded) = Repo::size_fixture().await; + std::fs::create_dir_all(repo.root.join("empty/deep")).unwrap(); + assert_eq!(repo.track(&repo.root).await, recorded); +} + +#[tokio::test] +async fn disabled_user_global_ignores_keep_bound_size_classification_available() { + let (repo, target, recorded) = Repo::size_fixture().await; + repo.git(&["config", "core.excludesfile", ""]); + let expected = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &expected) + .await; + assert_eq!(prepared.ignore.size_excluded, ["nested/large[1]"]); + assert_eq!( + repo.store(&repo.root) + .restore_git_worker(prepared, None) + .unwrap(), + ["file"] + ); + assert_eq!( + std::fs::read(repo.root.join("nested/large[1]")).unwrap(), + vec![b'x'; 5120] + ); +} + +#[tokio::test] +async fn changed_user_classification_config_after_preparation_refuses_native_restore() { + let (repo, target, recorded) = Repo::size_fixture().await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; + let ignore = repo._temp.path().join("classification-ignore"); + std::fs::write(&ignore, "nested/large[1]\n").unwrap(); + repo.git(&["config", "core.excludesfile", ignore.to_str().unwrap()]); + let store = repo.store(&repo.root); + assert!(store.restore_git_worker(prepared, None).is_err()); + assert_eq!(std::fs::read(repo.root.join("file")).unwrap(), b"after"); + assert!(store.recovery().unwrap().is_empty()); +} + #[tokio::test] async fn shrinking_size_excluded_file_after_preparation_refuses_native_restore() { let mut repo = Repo::new(); @@ -331,15 +583,12 @@ async fn native_git_index_proof_rejects_creation_replacement_and_changed_split_d } #[tokio::test] -async fn native_git_split_index_preparation_and_restore_preserve_user_index_metadata() { +async fn native_git_split_index_capture_preparation_and_restore_preserve_user_index_metadata() { use std::os::unix::fs::MetadataExt; let repo = Repo::new(); std::fs::write(repo.root.join("file"), "before").unwrap(); repo.git(&["add", "--", "file"]); repo.git(&["update-index", "--split-index"]); - let target = repo.track(&repo.root).await; - std::fs::write(repo.root.join("file"), "after").unwrap(); - let recorded = repo.track(&repo.root).await; let shared = std::fs::read_dir(repo.root.join(".git")) .unwrap() .map(|entry| entry.unwrap().path()) @@ -365,6 +614,10 @@ async fn native_git_split_index_preparation_and_restore_preserve_user_index_meta }) }; let before = observations(); + let target = repo.track(&repo.root).await; + std::fs::write(repo.root.join("file"), "after").unwrap(); + let recorded = repo.track(&repo.root).await; + assert_eq!(observations(), before); let prepared = repo .prepare(&repo.root, &target, &recorded, &recorded) .await; @@ -1537,6 +1790,14 @@ async fn native_inventory_reads_git_external_ignore_inputs_and_detects_rule_only std::fs::write(&global, b"file\n").unwrap(); assert!(matches!(inventory.verify(&store), Err(RestoreError::Stale))); std::fs::write(&global, b"*.global\n").unwrap(); + assert!(matches!( + store.restore_git_worker(prepared, None), + Err(RestoreError::Stale) + )); + let expected = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &expected) + .await; assert_eq!(store.restore_git_worker(prepared, None).unwrap(), ["file"]); assert_eq!( std::fs::read(repo.root.join("owner.info")).unwrap(), @@ -1654,14 +1915,14 @@ async fn capture_bound_policy_rejects_excluded_rule_changes_that_do_not_change_t let target = repo.track(&repo.root).await; std::fs::write(repo.root.join("file"), b"after").unwrap(); let recorded = repo.track(&repo.root).await; + let prepared = repo + .prepare(&repo.root, &target, &recorded, &recorded) + .await; std::fs::write( repo.root.join(".gitignore"), b"*.log\n# changed ignored rule input\n", ) .unwrap(); - let prepared = repo - .prepare(&repo.root, &target, &recorded, &recorded) - .await; assert!(!prepared.expected.images.contains_key(".gitignore")); let store = repo.store(&repo.root); assert!(matches!( diff --git a/crates/cyber-snapshot/src/copy/git_rules.rs b/crates/cyber-snapshot/src/copy/git_rules.rs index 4b142b93a..038095023 100644 --- a/crates/cyber-snapshot/src/copy/git_rules.rs +++ b/crates/cyber-snapshot/src/copy/git_rules.rs @@ -10,12 +10,70 @@ use std::path::{Path, PathBuf}; const INPUT_LIMIT: u64 = 1024 * 1024; const TOTAL_LIMIT: usize = 8 * 1024 * 1024; -#[derive(Clone, serde::Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] struct Binding { path: PathBuf, source: bool, hash: Option, + generation: Option, } +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub(crate) struct RuleProof { + root: PathBuf, + #[serde(serialize_with = "serialize_rule_bindings")] + bindings: Vec, + #[serde(skip)] + directories: Vec<(PathBuf, String)>, +} +fn serialize_rule_bindings( + bindings: &[Binding], + serializer: S, +) -> Result { + // Retain absence checks in memory, while new empty directories must not + // change checkpoint identity merely by adding absent .gitignore paths. + let relevant: Vec<_> = bindings + .iter() + .filter(|binding| !binding.source || binding.hash.is_some()) + .collect(); + serde::Serialize::serialize(&relevant, serializer) +} + +impl RuleProof { + pub(crate) fn verify_collection(&self) -> Result<(), String> { + let root = Dir::open_ambient_dir(&self.root, cap_std::ambient_authority()) + .map_err(|error| error.to_string())?; + for (path, generation) in &self.directories { + let mut directory = root.try_clone().map_err(|error| error.to_string())?; + for part in path.components() { + directory = directory + .open_dir_nofollow(part.as_os_str()) + .map_err(|error| error.to_string())?; + } + if dir_generation(&directory).map_err(|error| format!("{error:?}"))? != *generation { + return Err("Git classification directory changed".into()); + } + } + self.verify() + } + pub(crate) fn verify(&self) -> Result<(), String> { + let root = Dir::open_ambient_dir(&self.root, cap_std::ambient_authority()) + .map_err(|error| error.to_string())?; + for binding in &self.bindings { + let bytes = + read(&root, &binding.path, binding.source).map_err(|error| format!("{error:?}"))?; + if bytes + .as_ref() + .map(|value| blake3::hash(&value.bytes).to_hex().to_string()) + != binding.hash + || bytes.as_ref().map(|value| value.generation.clone()) != binding.generation + { + return Err("Git classification rule inputs changed".into()); + } + } + Ok(()) + } +} + pub(super) struct Rules { root: Dir, path: PathBuf, @@ -31,6 +89,9 @@ impl Rules { } fn open(worktree: &Path, prefix: &str, inputs: &IgnoreInputs) -> Result { inputs.config.verify().map_err(|_| RestoreError::Stale)?; + if let Some(classification) = &inputs.classification_rules { + classification.verify().map_err(|_| RestoreError::Stale)?; + } let mut rules = Self { root: Dir::open_ambient_dir(worktree, cap_std::ambient_authority()).map_err(failed)?, path: worktree.to_path_buf(), @@ -119,7 +180,9 @@ impl Rules { let bytes = read(&self.root, path, source)?; self.bytes = self .bytes - .checked_add(path.as_os_str().len() + bytes.as_ref().map_or(0, Vec::len)) + .checked_add( + path.as_os_str().len() + bytes.as_ref().map_or(0, |value| value.bytes.len()), + ) .ok_or_else(|| failed("Git rule input size overflow"))?; if self.bytes > TOTAL_LIMIT || self.bindings.len() >= 65536 { return Err(failed("Git rule inputs exceed limit")); @@ -129,9 +192,10 @@ impl Rules { source, hash: bytes .as_ref() - .map(|bytes| blake3::hash(bytes).to_hex().to_string()), + .map(|value| blake3::hash(&value.bytes).to_hex().to_string()), + generation: bytes.as_ref().map(|value| value.generation.clone()), }); - Ok(bytes) + Ok(bytes.map(|value| value.bytes)) } pub(super) fn fingerprint( &self, @@ -147,7 +211,7 @@ impl Rules { (left.source, &left.path, &left.hash).cmp(&(right.source, &right.path, &right.hash)) }); let bytes = serde_json::to_vec(&( - "git-policy-capture-v5", + "git-policy-capture-v6", &self.path, &self.inputs, bindings, @@ -180,6 +244,9 @@ impl Rules { } pub(super) fn verify(&self) -> Result<(), RestoreError> { self.verify_sizes()?; + if let Some(classification) = &self.inputs.classification_rules { + classification.verify().map_err(|_| RestoreError::Stale)?; + } self.inputs .config .verify() @@ -188,8 +255,9 @@ impl Rules { let bytes = read(&self.root, &binding.path, binding.source)?; if bytes .as_ref() - .map(|bytes| blake3::hash(bytes).to_hex().to_string()) + .map(|value| blake3::hash(&value.bytes).to_hex().to_string()) != binding.hash + || bytes.as_ref().map(|value| value.generation.clone()) != binding.generation { return Err(RestoreError::Stale); } @@ -197,7 +265,11 @@ impl Rules { Ok(()) } } -fn read(root: &Dir, path: &Path, source: bool) -> Result>, RestoreError> { +struct RuleRead { + bytes: Vec, + generation: String, +} +fn read(root: &Dir, path: &Path, source: bool) -> Result, RestoreError> { let result = if source { let mut directory = root.try_clone().map_err(failed)?; for part in path @@ -238,6 +310,7 @@ fn read(root: &Dir, path: &Path, source: bool) -> Result>, Restor Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(failed(error)), }; + let generation = super::ownership::generation(&file).map_err(RestoreError::Failed)?; let metadata = file.metadata().map_err(failed)?; if !metadata.is_file() || metadata.len() > INPUT_LIMIT { return Err(failed("Unsupported or over-limit Git rule input")); @@ -247,10 +320,12 @@ fn read(root: &Dir, path: &Path, source: bool) -> Result>, Restor .take(INPUT_LIMIT + 1) .read_to_end(&mut bytes) .map_err(failed)?; - if bytes.len() as u64 > INPUT_LIMIT { + if bytes.len() as u64 != metadata.len() + || super::ownership::generation(&file).map_err(RestoreError::Failed)? != generation + { return Err(failed("Git rule input grew beyond limit")); } - Ok(Some(bytes)) + Ok(Some(RuleRead { bytes, generation })) } fn failed(error: impl std::fmt::Display) -> RestoreError { RestoreError::Failed(fail(error)) @@ -262,6 +337,41 @@ pub(crate) fn capture_state( prefix: &str, inputs: &IgnoreInputs, ) -> Result { + let capture = collect(worktree, prefix, inputs, true)?; + Ok(super::git_capture::Capture { + policy: capture.rules.fingerprint(&capture.ignored)?, + files: capture.files, + }) +} + +pub(crate) fn classification_rules( + worktree: &Path, + prefix: &str, + inputs: &IgnoreInputs, +) -> Result { + let capture = collect(worktree, prefix, inputs, false).map_err(|error| format!("{error:?}"))?; + let mut bindings = capture.rules.bindings; + bindings.sort_by(|left, right| (left.source, &left.path).cmp(&(right.source, &right.path))); + let directories = capture + .marks + .into_iter() + .map(|(path, generation)| (Path::new(prefix).join(path), generation)) + .collect(); + let proof = RuleProof { + root: worktree.into(), + bindings, + directories, + }; + proof.verify_collection()?; + Ok(proof) +} + +fn collect( + worktree: &Path, + prefix: &str, + inputs: &IgnoreInputs, + images: bool, +) -> Result { let mut capture = PolicyCapture { rules: Rules::open(worktree, prefix, inputs)?, ignored: std::collections::BTreeSet::new(), @@ -270,6 +380,7 @@ pub(crate) fn capture_state( bytes: 0, files: std::collections::BTreeMap::new(), content_bytes: 0, + images, }; let mut directory = capture.rules.root.try_clone().map_err(failed)?; for part in Path::new(prefix).components() { @@ -290,10 +401,7 @@ pub(crate) fn capture_state( } } capture.rules.verify()?; - Ok(super::git_capture::Capture { - policy: capture.rules.fingerprint(&capture.ignored)?, - files: capture.files, - }) + Ok(capture) } struct PolicyCapture { rules: Rules, @@ -303,6 +411,7 @@ struct PolicyCapture { bytes: usize, files: std::collections::BTreeMap, content_bytes: usize, + images: bool, } impl PolicyCapture { fn walk( @@ -366,7 +475,7 @@ impl PolicyCapture { return Err(failed("Embedded Git policy proof is unavailable")); } self.walk(&child, &relative, prefix, depth + 1)?; - } else { + } else if self.images { if self.files.len() >= 4096 { return Err(failed("Git raw capture file count exceeds limit")); } diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index 377a8e418..b44103459 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -9,6 +9,8 @@ use sha1::{Digest, Sha1}; mod catalog; #[cfg(any(target_os = "macos", target_os = "linux"))] mod config; +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod eligibility; #[path = "git_history.rs"] pub(crate) mod history; mod images; @@ -37,6 +39,7 @@ pub(crate) struct Shadow { /// Environment variables that would redirect git away from the shadow repository. pub(crate) const GIT_ENV: &[&str] = &[ "GIT_DIR", + "GIT_CONFIG", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", @@ -205,8 +208,13 @@ impl Shadow { pub async fn track(&self, ignore: &[String], max_bytes: u64) -> Result { self.ensure_init().await?; - let skipped = self.large_untracked(max_bytes).await; - self.write_excludes(ignore, max_bytes, &skipped)?; + #[cfg(any(target_os = "macos", target_os = "linux"))] + self.write_capture_excludes(ignore, max_bytes).await?; + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let skipped = self.large_untracked(max_bytes).await; + self.write_excludes(ignore, max_bytes, &skipped)?; + } #[cfg(any(target_os = "macos", target_os = "linux"))] let capture_before = self.capture_git_state(ignore, max_bytes).await; let excluded = self.omitted().await?; @@ -277,52 +285,33 @@ impl Shadow { /// Untracked files (per the user's repository) over the size limit. async fn large_untracked(&self, max_bytes: u64) -> Vec { - let mut cmd = tokio::process::Command::new("git"); - for name in GIT_ENV { - cmd.env_remove(name); - } - let out = cmd - .arg("--literal-pathspecs") - .arg("-C") - .arg(&self.worktree) - .args([ - "ls-files", - "--others", - "--exclude-standard", - "-z", - "--", - &self.pathspec, - ]) - .output() - .await; - let Ok(out) = out else { return Vec::new() }; - out.stdout + let mut command = self.command(&[ + "ls-files", + "--others", + "--exclude-standard", + "-z", + "--", + &self.pathspec, + ]); + command + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env("GIT_OPTIONAL_LOCKS", "0"); + let Ok(bytes) = + Self::bounded_command(command, 8 * 1024 * 1024, std::time::Duration::from_secs(10)) + .await + else { + return Vec::new(); + }; + bytes .split(|b| *b == 0) .filter(|p| !p.is_empty()) + .take(65536) .map(|p| String::from_utf8_lossy(p).to_string()) .filter(|p| std::fs::metadata(self.worktree.join(p)).is_ok_and(|m| m.len() > max_bytes)) .collect() } - /// The user's `info/exclude`, `snapshots.ignore` and the skipped large files. - #[cfg(any(target_os = "macos", target_os = "linux"))] - fn write_excludes( - &self, - ignore: &[String], - max_untracked: u64, - skipped: &[String], - ) -> Result<(), String> { - crate::copy::generated_rules::write( - &self.git_dir, - &self.common_dir, - &prepared::CaptureSettings { - ignore: ignore.to_vec(), - max_untracked, - }, - skipped, - ) - } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] fn write_excludes( &self, diff --git a/crates/cyber-snapshot/src/git/config.rs b/crates/cyber-snapshot/src/git/config.rs index a97263495..704629226 100644 --- a/crates/cyber-snapshot/src/git/config.rs +++ b/crates/cyber-snapshot/src/git/config.rs @@ -11,6 +11,44 @@ pub(super) struct Inventory { } impl Inventory { + pub(super) fn frozen_config(&self) -> Result, String> { + let mut text = String::new(); + for (_, key, value) in records(&self.values)? { + if include_key(key) { + continue; + } + let (section, rest) = key.split_once('.').ok_or("Invalid frozen Git config key")?; + let (subsection, variable) = rest + .rsplit_once('.') + .map_or((None, rest), |(subsection, variable)| { + (Some(subsection), variable) + }); + if !config_name(section) || !config_name(variable) { + return Err("Unsupported frozen Git config key".into()); + } + text.push('['); + text.push_str(section); + if let Some(subsection) = subsection { + if subsection.chars().any(char::is_control) { + return Err("Unsupported frozen Git config subsection".into()); + } + text.push(' '); + text.push_str("e(subsection)?); + } + text.push_str("]\n\t"); + text.push_str(variable); + if let Some(value) = value { + text.push_str(" = "); + text.push_str("e(value)?); + } + text.push('\n'); + if text.len() > 1024 * 1024 { + return Err("Frozen Git configuration exceeds limit".into()); + } + } + Ok(text.into_bytes()) + } + pub(super) fn selected(&self, key: &str) -> Result>, String> { Ok(records(&self.values)? .into_iter() @@ -22,16 +60,36 @@ impl Inventory { impl Shadow { pub(super) async fn config_inputs(&self) -> Result<(ConfigInputs, Inventory), String> { + self.config_inputs_profile(true).await + } + + pub(super) async fn user_config_inputs(&self) -> Result<(ConfigInputs, Inventory), String> { + self.config_inputs_profile(false).await + } + + async fn config_inputs_profile( + &self, + private: bool, + ) -> Result<(ConfigInputs, Inventory), String> { let environment = environment_digest()?; let contexts = vec![self.worktree.clone(), self.worktree.join(&self.git_dir)]; - let roots = self.config_roots().await?; - let original = capture(roots.clone(), environment.clone(), contexts.clone()).await?; + let mut roots = self.config_roots().await?; + if !private { + roots.push(self.common_dir.join("config")); + } + let original = capture( + roots.clone(), + environment.clone(), + contexts.clone(), + private, + ) + .await?; let inventory = self.config_inventory().await?; let paths = roots .into_iter() .chain(inventory.paths.iter().cloned()) .collect(); - let proof = capture(paths, environment, contexts).await?; + let proof = capture(paths, environment, contexts, private).await?; if !proof.retains(&original) { return Err("Git configuration roots changed during resolution".into()); } @@ -82,9 +140,6 @@ impl Shadow { paths.insert(self.worktree.join(path)); } } - if let Some(path) = std::env::var_os("GIT_CONFIG").filter(|path| !path.is_empty()) { - paths.insert(self.worktree.join(path)); - } Ok(paths.into_iter().collect()) } @@ -197,10 +252,37 @@ impl Shadow { } } +fn config_name(name: &str) -> bool { + name.as_bytes().first().is_some_and(u8::is_ascii_alphabetic) + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + +fn quote(value: &str) -> Result { + let mut text = String::from("\""); + for character in value.chars() { + match character { + '\\' => text.push_str("\\\\"), + '"' => text.push_str("\\\""), + '\n' => text.push_str("\\n"), + '\t' => text.push_str("\\t"), + '\u{0008}' => text.push_str("\\b"), + character if character.is_control() => { + return Err("Unsupported frozen Git config value".into()); + } + character => text.push(character), + } + } + text.push('"'); + Ok(text) +} + async fn capture( paths: Vec, environment: String, contexts: Vec, + private: bool, ) -> Result { tokio::task::spawn_blocking(move || { let mut proof = ConfigInputs::capture(paths, environment)?; @@ -209,7 +291,9 @@ async fn capture( .ok_or("Missing private Git configuration context")? .clone(); proof.bind_contexts(contexts)?; - proof.bind_repository(repository)?; + if private { + proof.bind_repository(repository)?; + } Ok(proof) }) .await @@ -319,6 +403,59 @@ mod tests { assert_eq!(inventory.selected("core.ignorecase").unwrap(), Some(None)); } + #[test] + fn frozen_configuration_round_trips_values_without_replaying_includes() { + let entries = [ + ("include.path", Some("never-read")), + ( + "custom.quoted.sub.value", + Some(" quotes \" \\ \n\t\u{0008}"), + ), + ("custom.flag", None), + ("custom.empty", Some("")), + ]; + let mut values = Vec::new(); + for (key, value) in entries { + values.extend_from_slice(format!("file:a\0{key}").as_bytes()); + if let Some(value) = value { + values.push(b'\n'); + values.extend_from_slice(value.as_bytes()); + } + values.push(0); + } + let inventory = Inventory { + values: values.to_vec(), + paths: BTreeSet::new(), + }; + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("config"); + std::fs::write(&path, inventory.frozen_config().unwrap()).unwrap(); + let output = std::process::Command::new("git") + .args(["config", "--file"]) + .arg(path) + .args(["--null", "--show-origin", "--list"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let actual: Vec<_> = records(&output.stdout) + .unwrap() + .into_iter() + .map(|(_, key, value)| (key, value)) + .collect(); + let expected: Vec<_> = records(&values) + .unwrap() + .into_iter() + .filter(|(_, key, _)| !include_key(key)) + .map(|(_, key, value)| (key, value)) + .collect(); + assert_eq!(actual, expected); + assert!(quote("unsupported\rvalue").is_err()); + } + #[tokio::test] async fn expansion_uses_supplied_values_instead_of_reselecting_mutable_settings() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/cyber-snapshot/src/git/eligibility.rs b/crates/cyber-snapshot/src/git/eligibility.rs new file mode 100644 index 000000000..84c45f8d7 --- /dev/null +++ b/crates/cyber-snapshot/src/git/eligibility.rs @@ -0,0 +1,395 @@ +//! Capture size classification from frozen user index and resolved config inputs. +use super::{Shadow, config::Inventory, user_index::UserIndex}; +use crate::copy::config_inputs::ConfigInputs; +use crate::git::prepared::CaptureSettings; +use std::path::Path; + +struct Context { + inputs: ConfigInputs, + index: UserIndex, + user: Shadow, + config: Inventory, + rules: crate::copy::git_rules::RuleProof, + global: Option, +} + +impl Shadow { + async fn classification_context(&self) -> Result { + let index = self.user_index().await?; + let mut user = self.clone(); + user.git_dir = index.directory.clone(); + let (mut inputs, config) = user.user_config_inputs().await?; + inputs.merge(index.inputs.clone())?; + let global = user_global(&user, &config).await?; + let paths = global + .iter() + .cloned() + .chain(std::iter::once(user.common_dir.join("info/exclude"))) + .collect(); + inputs.merge(ConfigInputs::capture_files(paths)?)?; + let raw = config + .selected("core.ignorecase")? + .map_or("false", |value| value.unwrap_or("true")); + let case_insensitive = user.config_boolean("core.ignorecase", raw).await?; + let policy = crate::git::prepared::IgnoreInputs { + config: inputs.clone(), + info: user.common_dir.join("info/exclude"), + global: global.clone(), + case_insensitive, + private_roots: self.private_roots.clone(), + settings: CaptureSettings { + ignore: vec![], + max_untracked: u64::MAX, + }, + size_excluded: vec![], + classification_rules: None, + }; + let prefix = if self.pathspec == "." { + "" + } else { + &self.pathspec + }; + let worktree = self.worktree.clone(); + let prefix = prefix.to_owned(); + let rules = tokio::task::spawn_blocking(move || { + crate::copy::git_rules::classification_rules(&worktree, &prefix, &policy) + }) + .await + .map_err(|error| error.to_string())??; + user.verify_config_inputs(&inputs, &config).await?; + Ok(Context { + inputs, + index, + user, + config, + rules, + global, + }) + } + + pub(super) async fn write_capture_excludes( + &self, + ignore: &[String], + max_untracked: u64, + ) -> Result<(), String> { + let classified = self.classify_untracked(max_untracked).await; + match classified { + Ok(classified) => { + classified.rules.verify_collection()?; + crate::copy::generated_rules::write( + &self.git_dir, + &self.common_dir, + &CaptureSettings { + ignore: ignore.to_vec(), + max_untracked, + }, + &classified.skipped, + Some((&classified.fingerprint()?, &classified.inputs)), + )?; + classified.rules.verify_collection() + } + Err(_) => { + let skipped = self.large_untracked(max_untracked).await; + crate::copy::generated_rules::write( + &self.git_dir, + &self.common_dir, + &CaptureSettings { + ignore: ignore.to_vec(), + max_untracked, + }, + &skipped, + None, + ) + } + } + } + + pub(super) async fn classify_untracked( + &self, + max_untracked: u64, + ) -> Result { + let context = self.classification_context().await?; + // Git refreshes shared-index timestamps while reading. Only disposable + // copies may be passed to its index reader, including split dependencies. + let temporary = tempfile::Builder::new() + .prefix(".eligibility-") + .tempdir_in(&self.git_dir) + .map_err(|error| error.to_string())?; + let directory = temporary.path().to_path_buf(); + let (context, copies) = tokio::task::spawn_blocking(move || { + let copies = context.materialize(&directory)?; + Ok::<_, String>((context, copies)) + }) + .await + .map_err(|error| error.to_string())??; + let global_argument = format!( + "core.excludesfile={}", + temporary.path().join("global-ignore").display() + ); + let mut command = self.command(&[ + "-c", + &global_argument, + "-c", + "core.untrackedcache=false", + "ls-files", + "--others", + "--exclude-standard", + "-z", + "--", + &self.pathspec, + ]); + command + .env("GIT_DIR", temporary.path()) + .env("GIT_OPTIONAL_LOCKS", "0") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_SYSTEM", "/dev/null") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_COUNT", "0") + .env_remove("GIT_CONFIG_PARAMETERS") + .env_remove("GIT_CONFIG"); + let bytes = + Self::bounded_command(command, 8 * 1024 * 1024, std::time::Duration::from_secs(10)) + .await?; + context + .user + .verify_config_inputs(&context.inputs, &context.config) + .await?; + let worktree = self.worktree.clone(); + let private_roots = self.private_roots.clone(); + tokio::task::spawn_blocking(move || { + let skipped = size_candidates(&worktree, &private_roots, &bytes, max_untracked)?; + context.inputs.verify()?; + copies.verify()?; + context.rules.verify_collection()?; + Ok(Classified { + skipped, + inputs: context.inputs, + rules: context.rules, + }) + }) + .await + .map_err(|error| error.to_string())? + } +} + +pub(super) struct Classified { + pub skipped: Vec, + pub inputs: ConfigInputs, + pub rules: crate::copy::git_rules::RuleProof, +} +impl Classified { + pub(super) fn fingerprint(&self) -> Result { + let bytes = serde_json::to_vec(&( + "git-size-classification-v1", + self.inputs.fingerprint()?, + &self.rules, + )) + .map_err(|error| error.to_string())?; + Ok(blake3::hash(&bytes).to_hex().to_string()) + } +} + +async fn user_global( + user: &Shadow, + config: &Inventory, +) -> Result, String> { + let default = std::env::var_os("XDG_CONFIG_HOME") + .filter(|value| !value.is_empty()) + .map(|path| std::path::PathBuf::from(path).join("git/ignore")) + .unwrap_or_else(|| std::path::PathBuf::from("~/.config/git/ignore")); + let raw = match config.selected("core.excludesfile")? { + Some(value) => value.ok_or("Git classification ignore path lacks value")?, + None => default + .to_str() + .ok_or("Non-UTF-8 classification ignore path")?, + }; + let path = user + .expand_config_paths(&[raw]) + .await? + .pop() + .ok_or("Missing classification ignore path")?; + Ok((!path.as_os_str().is_empty()).then(|| user.worktree.join(path))) +} + +impl Context { + fn materialize(&self, directory: &Path) -> Result { + for child in ["objects/info", "refs"] { + std::fs::create_dir_all(directory.join(child)).map_err(|error| error.to_string())?; + } + // Includes have already been resolved in the user's real branch/gitdir + // context. Store values privately rather than placing secrets in argv. + private_write(directory.join("config"), self.config.frozen_config()?) + .map_err(|error| error.to_string())?; + private_write( + directory.join("global-ignore"), + self.global + .as_ref() + .map(|path| self.inputs.bytes(path)) + .transpose()? + .unwrap_or_default(), + ) + .map_err(|error| error.to_string())?; + std::fs::create_dir(directory.join("info")).map_err(|error| error.to_string())?; + private_write( + directory.join("info/exclude"), + self.inputs + .bytes(&self.user.common_dir.join("info/exclude"))?, + ) + .map_err(|error| error.to_string())?; + let head = self.inputs.bytes(&self.index.directory.join("HEAD"))?; + private_write(directory.join("HEAD"), head).map_err(|error| error.to_string())?; + for path in &self.index.files { + if let Some(bytes) = self.inputs.optional_bytes(path)? { + let name = path.file_name().ok_or("Missing frozen index filename")?; + private_write(directory.join(name), bytes).map_err(|error| error.to_string())?; + } + } + let objects = self.user.common_dir.join("objects"); + let path = objects.to_str().ok_or("Non-UTF-8 Git object directory")?; + if path.chars().any(char::is_control) { + return Err("Unsupported frozen Git object directory".into()); + } + let alternate = serde_json::to_string(path).map_err(|error| error.to_string())?; + private_write( + directory.join("objects/info/alternates"), + format!("{alternate}\n"), + ) + .map_err(|error| error.to_string())?; + let mut paths = [ + "config", + "global-ignore", + "info/exclude", + "HEAD", + "objects/info/alternates", + ] + .map(|name| directory.join(name)) + .to_vec(); + for source in &self.index.files { + if self.inputs.optional_bytes(source)?.is_some() { + paths.push( + directory.join(source.file_name().ok_or("Missing copied index filename")?), + ); + } + } + let mut layout = paths + .iter() + .map(|path| { + Ok(( + path.strip_prefix(directory) + .map_err(|error| error.to_string())? + .to_path_buf(), + false, + )) + }) + .collect::, String>>()?; + for child in ["objects", "objects/info", "refs", "info"] { + layout.insert(child.into(), true); + } + let copies = PrivateCopies { + inputs: ConfigInputs::capture_files(paths)?, + root: directory.into(), + layout, + }; + self.inputs.verify()?; + copies.verify()?; + Ok(copies) + } +} + +struct PrivateCopies { + inputs: ConfigInputs, + root: std::path::PathBuf, + layout: std::collections::BTreeMap, +} +impl PrivateCopies { + fn verify(&self) -> Result<(), String> { + use cap_fs_ext::DirExt; + self.inputs.verify_private_index_copies()?; + let root = cap_std::fs::Dir::open_ambient_dir(&self.root, cap_std::ambient_authority()) + .map_err(|error| error.to_string())?; + let mut pending = vec![(root, std::path::PathBuf::new())]; + let mut found = 0; + while let Some((directory, prefix)) = pending.pop() { + for entry in directory.entries().map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let name = entry.file_name(); + let path = prefix.join(&name); + let kind = entry.file_type().map_err(|error| error.to_string())?; + if self.layout.get(&path) != Some(&kind.is_dir()) + || (!kind.is_dir() && !kind.is_file()) + { + return Err("Private Git classifier namespace changed".into()); + } + found += 1; + if kind.is_dir() { + pending.push(( + directory + .open_dir_nofollow(name) + .map_err(|error| error.to_string())?, + path, + )); + } + } + } + if found != self.layout.len() { + return Err("Private Git classifier namespace is incomplete".into()); + } + Ok(()) + } +} + +fn private_write(path: impl AsRef, bytes: impl AsRef<[u8]>) -> std::io::Result<()> { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(path)? + .write_all(bytes.as_ref()) +} + +fn size_candidates( + worktree: &Path, + private_roots: &[std::path::PathBuf], + bytes: &[u8], + limit: u64, +) -> Result, String> { + if !bytes.is_empty() && bytes.last() != Some(&0) { + return Err("Unterminated Git untracked listing".into()); + } + let mut skipped = std::collections::BTreeSet::new(); + for (count, path) in bytes + .split(|byte| *byte == 0) + .filter(|path| !path.is_empty()) + .enumerate() + { + if count >= 65536 { + return Err("Git untracked listing exceeds entry limit".into()); + } + let path = std::str::from_utf8(path).map_err(|_| "Non-UTF-8 Git untracked path")?; + super::images::validate_path(path)?; + if private_roots + .iter() + .any(|root| worktree.join(path).starts_with(root)) + || Path::new(path).components().any(|part| { + matches!( + part.as_os_str().to_str(), + Some(".cyber" | ".claude" | ".git") + ) + }) + { + continue; + } + let metadata = + std::fs::symlink_metadata(worktree.join(path)).map_err(|error| error.to_string())?; + if metadata.is_file() && metadata.len() > limit { + skipped.insert(path.to_owned()); + } + if skipped.len() > 4096 { + return Err("Git size exclusion count exceeds limit".into()); + } + } + Ok(skipped.into_iter().collect()) +} diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index 56fd1af7a..102dba20d 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -73,6 +73,7 @@ pub(crate) struct IgnoreInputs { pub private_roots: Vec, pub settings: CaptureSettings, pub size_excluded: Vec, + pub classification_rules: Option, } pub(crate) struct PreparedRestore { @@ -126,7 +127,13 @@ impl Shadow { let generated = crate::copy::generated_rules::observe(&self.git_dir, &self.common_dir, settings)?; config.merge(generated.inputs)?; - config.merge(self.user_index_inputs().await?)?; + let classified = self.classify_untracked(settings.max_untracked).await?; + if generated.classification.as_deref() != Some(classified.fingerprint()?.as_str()) + || generated.size_excluded != classified.skipped + { + return Err("Generated Git size classification changed or lacks provenance".into()); + } + config.merge(classified.inputs)?; Ok(IgnoreInputs { config, info: self.git_dir.join("info/exclude"), @@ -135,10 +142,12 @@ impl Shadow { private_roots: self.private_roots.clone(), settings: settings.clone(), size_excluded: generated.size_excluded, + classification_rules: Some(classified.rules), }) } - /// No source reads, Git index writes or merge subprocesses occur here. + /// Policy collection reads rules and metadata; immutable images supply bytes. + /// No source image reads, user-index writes or merge subprocesses occur here. pub(crate) async fn prepare_restore( &self, request: &SnapshotRestoreRequest, diff --git a/crates/cyber-snapshot/src/git/user_index.rs b/crates/cyber-snapshot/src/git/user_index.rs index 651eb0167..050388f1b 100644 --- a/crates/cyber-snapshot/src/git/user_index.rs +++ b/crates/cyber-snapshot/src/git/user_index.rs @@ -9,8 +9,14 @@ struct Layout { index: PathBuf, } +pub(super) struct UserIndex { + pub inputs: ConfigInputs, + pub directory: PathBuf, + pub files: Vec, +} + impl Shadow { - pub(super) async fn user_index_inputs(&self) -> Result { + pub(super) async fn user_index(&self) -> Result { // Observe discovery routing before Git resolves any metadata paths. let marker = self.worktree.join(".git"); let metadata = std::fs::symlink_metadata(&marker).map_err(|error| error.to_string())?; @@ -33,7 +39,9 @@ impl Shadow { layout.directory.join("commondir"), layout.directory.join("gitdir"), ]; - paths.extend(shared_indexes(&layout.directory)?); + let mut files = vec![layout.index.clone()]; + files.extend(shared_indexes(&layout.directory)?); + paths.extend(files.iter().skip(1).cloned()); let mut index = ConfigInputs::capture_files(paths)?; index.bind_contexts(vec![layout.directory.clone()])?; proof.merge(index)?; @@ -41,7 +49,11 @@ impl Shadow { return Err("User Git index layout changed during collection".into()); } proof.verify()?; - Ok(proof) + Ok(UserIndex { + inputs: proof, + directory: layout.directory, + files, + }) } async fn user_index_layout(&self) -> Result { diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 4bcd1db70..316d267d7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3534,3 +3534,115 @@ The GitHub refresh still reports #93 and #92 open and origin/main remains `db9e171e`, already contained in this branch. PR #115 concerns model fallback and is outside this recovery increment; it was left untouched. No goal PR/push/merge or parent completion is claimed. + +### Bound size-classification generation (2026-11-07) + +A new real-repository test reproduced another missing guard: changing the user's +classification config after native preparation still allowed source restoration. +Its initial refusal assertion failed on this branch; there is no unchanged-main +attribution. New version-2 generated-rule recipes retain a classification digest +covering the observed actual user index/routing, resolved user configuration and +rule observations. Fresh capture/preparation repeats bounded classification and +requires both matching digest and exact skipped names before granting proof. +Version-1 recipes remain canonical-readable without invented classification proof. +The native policy frame is v6, so older checkpoint proof is not promoted. + +The classifier materializes only observed index/shared-index bytes in a private +temporary admin directory. Active config includes are resolved in the user's real +branch/gitdir context; ordered effective values (including null/empty and quoted +values) are serialized into a private config without re-evaluating includes there. +Global/info excludes are copied from bound observations, and source rule reads +remain no-follow. Config secrets stay out of command arguments and checkpoints. +The generic user-config profile retains roots, active/absent includes, HEAD and +physical context without applying the private shadow's empty-reference profile to +a real user repository. GIT_CONFIG is now cleared with repository redirects: it +redirects config commands rather than ordinary repository reads and could otherwise +make internal setup modify an unrelated file. A subprocess regression verifies +native capture/restore and preservation of that external config. + +Private copied files use create-new/no-follow mode 0600 beneath a private temporary +directory. They are verified again after Git reads them. Only disposable shared- +index timestamps may refresh; byte digest, inode, stable attributes and physical +route still have to match. Original index/config inputs retain strict generation +verification. Rule reads now bracket content reads with generation checks, and +classification retains both bytes' hashes and generations. Directory marks guard +the collection window but stay out of checkpoint fingerprints and native handoff +verification, preserving harmless empty-directory changes. Existing native +namespace validation still runs. Blocking rule scans, metadata census and private +index materialization run outside async executor threads. Metadata copies retain +the existing 1 MiB/file and 8 MiB/aggregate input bounds; rule traversal, output, +entry count and process deadlines remain bounded. The canonical fallback's Git +listing now uses the bounded kill-and-await runner too. + +Regression evidence includes original-config changes, effective gitdir/onbranch +includes in their original context, null/quoted/multiline config round trips, +disabled global excludes and original split-index metadata/bytes preserved across +capture, preparation and native restoration. An isolated real Git wrapper changes +the original index or the disposable private config after classification has read +its inputs. Both cases retain canonical-readable snapshots with explicit missing +native proof and no native journal/source effects. The original-index case still +captures its now-tracked large file; private-copy tampering retains the ordinary +size omission. This wrapper injects deterministic interference; it is not an OS +process-death cut. + +Review corrected an overly strict first pass that rejected harmless new empty +directories. Generation checks are retained where they detect collection races, +while native planning uses existing eligibility/namespace rules. An older rule- +change test now keeps a prepared handoff before the edit to exercise native +refusal. A rule changed and restored to identical bytes has a new generation and +requires a fresh expected capture; its positive then restores correctly. The +conditional-include fixture initially used a trailing slash matching descendants +rather than the exact gitdir, and was corrected with an actual Git selection +assertion. A test initially attempted process-global environment mutation, rejected +by the workspace unsafe-code lint; it now derives the root from the child command's +existing worktree environment. Clippy's collapsible-if finding was fixed without +allowances. Code-review skill inspection covered original-vs-private proof +separation, no secret argv, recipe/version compatibility, bounded child cleanup, +no-follow paths, private-copy interference and preservation of positive behavior. + +Local snapshot-library acceptance passed 136 tests with six intentionally ignored +subprocess fixtures in 85.64 seconds. A subsequently added disabled-global-excludes +positive passed separately in 4.24 seconds; production code is the same. Strict +OpenSpec passed 117 items. Remaining check results are recorded below. All managed +Git source admission and M2–M8/parent acceptance remain disabled/open. Full general +configuration/ABA/dependency and sparse/filter/limit combinations, native runtime/ +checkout/recovery, partial construction, cleanup of metadata/CAS and orphan private +classification directories after process death, and full native platform/cut +acceptance remain required. These checks do not prove the two issues complete. + +Final namespace review also requires an exact disposable admin inventory before +and after Git: only the constructed regular files and directories may exist, and +links/unknown routing files refuse without traversal. The interference fixture now +also inserts an unexpected commondir file, and removes its own output scratch file +before verification so that each intended guard is exercised independently. The +copied-file proof retains stable inode/attribute identity separately from full +metadata generation; its shared-index timestamp exception is used only on these +private copies, never on original user inputs. No unresolved critical/major finding +remains in this bounded classification increment; broader compatibility, process- +death cleanup and activation acceptance remain open as listed above. + +The final complete package invocation passed all 137 library cases (six ignored +subprocess fixtures), 18 file-copy integration cases in 1.23 seconds and 22 Git +integration cases in 23.58 seconds (one ignored subprocess fixture). Library time +was 89.01 seconds. The Windows-only platform binary and doctests selected zero +cases on this host and provide no additional platform/doc execution evidence. +Final all-target snapshot Clippy with -D warnings, format/diff checks, generated +SDK consistency, strict OpenSpec (117 items) and spec lint (0 errors/25 warnings) +passed. All Cargo tests/Clippy used CARGO_INCREMENTAL=0 and --locked. No full +workspace, CLI/TUI rerun, SDK execution or source-bound native CI/platform matrix +ran for this increment. No push/PR/merge or parent/issue completion is claimed. + +A final compatibility regression then reproduced changed checkpoint IDs after +adding only empty directories. Classification retained absent .gitignore bindings +for those new directories in its serialized digest. They now stay available for +verification but are filtered from serialization using the same source-absence +rule as native policy capture. This preserves both ordinary checkpoint identity +and classification identity while retaining mutation detection. The regression +failed before that correction and passes afterward; no baseline attribution is +made. The final package rerun supersedes the earlier counts above: 138 library +cases passed in 86.93 seconds, plus 18 file-copy cases in 1.14 seconds and 22 Git +cases in 23.56 seconds. Six library and one integration subprocess fixtures remain +intentionally excluded from direct top-level selection. Windows-only/doc binaries +again selected zero. Final all-target Clippy, format/diff, strict OpenSpec (117), +lint (0/25) and SDK generation consistency passed. No further scope or acceptance +claim changes: source admission and parent tasks remain open as stated above. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index b2871cb02..88c3a6147 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -766,6 +766,38 @@ recipe's classification was derived from the same user index/configuration durin its original generation; that separate capture-provenance requirement and full Git runtime acceptance remain necessary before managed Git admission. +### Capture provenance of generated size exclusions + +A version-2 generated-rule recipe SHALL retain a digest of the exact observed +user index/routing and resolved configuration inputs used to classify untracked +files. The classifier SHALL read private copies of observed index/shared-index +bytes, replay effective configuration values after resolving includes in their +original context, and avoid re-evaluating conditional includes in the private +context. It SHALL preserve the user's index metadata and bytes. The original +input observations SHALL be verified before/after classification and recipe +publication. Native capture/preparation SHALL compare fresh input proof with the +recipe digest and retain it for native revalidation without Git subprocesses. +Legacy/unproven recipes SHALL remain canonical-readable without native authority. +Classifier output, arguments, metadata copies and subprocess lifetime SHALL be +bounded. Unsupported or failed collection SHALL visibly withhold native proof +while preserving canonical capture. Current size/type revalidation and historical +exclusion constraints remain mandatory. This does not itself activate managed Git +source admission or complete filter/runtime/platform acceptance. + +Classification rule observations SHALL detect changed generations as well as +bytes. Directory-generation marks SHALL fence only the classification window; +they SHALL NOT turn harmless empty-directory additions into permanent checkpoint +conflicts. Disposable private files SHALL be created without following links and +with mode 0600 beneath a private temporary directory, then revalidated after Git. +The complete private admin inventory SHALL contain only the constructed regular +files and directories; links and unknown routing entries SHALL refuse. Only +timestamps of disposable shared-index copies may refresh; their content, +inode, stable attributes and route SHALL still match. Original input checks stay +strict. Configuration values SHALL remain in private files, not command arguments. +GIT_CONFIG, which redirects configuration commands rather than the repository's +ordinary reads, SHALL be cleared with repository redirects so internal setup +cannot modify an unrelated file. + ### Typed recovery CLI dispatch `cyber sessions recovery ` SHALL list retained journals without inference. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 241fb6b21..21ea76858 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -403,3 +403,32 @@ - **THEN** the current capture SHALL include its exact raw image and remove the expired generated size exclusion - **AND** target/recorded historical omissions SHALL continue to protect that file from fabricated deletion/restoration - **AND** this size revalidation SHALL NOT imply original classification provenance, complete user config dependencies or managed Git runtime acceptance + + +#### Scenario: Generated size classification retains capture provenance +- **GIVEN** native Git capture classifies untracked files against a user's index and resolved configuration +- **WHEN** the generated-rule recipe is published +- **THEN** a version-2 recipe SHALL retain the bounded classification-input digest, and fresh capture/preparation SHALL require matching inputs and skipped names +- **AND** classification SHALL use private index/shared-index copies, effective configuration resolved in the original gitdir/branch context, and retained rule observations +- **AND** original index/shared-index bytes and metadata SHALL remain unchanged on the supported bound path + +#### Scenario: Classification or private copied inputs change during collection +- **WHEN** the original index/config/rules change during classification or a private copied config/index changes unexpectedly +- **THEN** the result SHALL NOT grant native authority, and canonical-readable fallback SHALL explicitly report unavailable native proof +- **AND** Git's timestamp refresh on disposable shared-index copies SHALL NOT waive their content, inode, attribute or route checks, or any strict original-input check + +#### Scenario: Git configuration command override cannot redirect snapshot writes +- **GIVEN** the process environment contains GIT_CONFIG pointing at an unrelated file +- **WHEN** internal snapshot Git configuration is initialized or queried +- **THEN** the command-only override SHALL be cleared with repository redirects, internal writes SHALL target the shadow, and the unrelated file SHALL remain unchanged + +#### Scenario: Historical generated rules lack classification provenance +- **WHEN** an existing version-1 generated recipe has no classification digest +- **THEN** it SHALL remain readable for canonical capture but SHALL NOT be promoted into native classification proof +- **AND** a fresh supported capture SHALL regenerate current proof without changing historical checkpoint image identities or exclusion protections + + +#### Scenario: Empty directories preserve checkpoint identity +- **GIVEN** captured file images, effective configuration, ignores and size-excluded names are unchanged +- **WHEN** harmless empty directories add only absent source .gitignore observations +- **THEN** those absence checks SHALL remain available during verification without changing serialized classification provenance or checkpoint identity diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index fd9bcf482..701193a6e 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -573,3 +573,17 @@ cases and still-oversized/fresh-capture historical-exclusion positives are cover Original classification provenance at generation, full configuration/filter/limit compatibility and runtime/checkout/recovery/cleanup/platform acceptance remain required; managed Git source admission and parent task acceptance stay disabled. + +Classification-provenance continuation: version-2 generated recipes now bind +observed user index/routing, resolved user configuration and rule inputs. A private +classifier uses copied index/shared-index bytes and effective configuration whose +includes were resolved in the original branch/gitdir context; original metadata +stays strict. Private copied files have mode 0600 and post-read verification, with +only disposable shared-index timestamp refresh allowed. Fresh capture/preparation +requires matching provenance and skipped names; legacy/unproven recipes retain +canonical readability without native authority. GIT_CONFIG command redirection is +cleared to prevent internal configuration writes reaching an unrelated file. +Supported capture/preparation/restore and mutation-during-classification positives +and denials are covered. Complete configuration/ABA/filter/limit compatibility, +managed Git runtime/checkout/recovery/cleanup and source-bound platform acceptance +remain open; managed source admission and all parent acceptance stay disabled. From b2f35a239ff32819fea667d4eed9ba4219a67f31 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 14:13:04 -0400 Subject: [PATCH 48/87] Exercise owned Git restores through runtime workers --- .github/workflows/non-git-snapshot-proof.yml | 21 +- crates/cyber-app/Cargo.toml | 3 + .../cyber-app/tests/snapshot_git_runtime.rs | 308 ++++++++++++++++++ crates/cyber-server/src/workspace.rs | 40 ++- crates/cyber-snapshot/Cargo.toml | 4 + crates/cyber-snapshot/src/copy/git/tests.rs | 72 ++-- crates/cyber-snapshot/src/lib.rs | 40 ++- .../implementation.md | 75 +++++ .../managed-recovery.md | 24 ++ .../specs/snapshots-checkpoints/spec.md | 19 ++ .../complete-daily-coding-surfaces/tasks.md | 13 + 11 files changed, 564 insertions(+), 55 deletions(-) create mode 100644 crates/cyber-app/tests/snapshot_git_runtime.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index d388f5dd2..8bdaa9140 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -1,8 +1,16 @@ -name: Non-Git snapshot native proof +name: Snapshot native proof on: pull_request: paths: - crates/cyber-snapshot/** + - crates/cyber-app/Cargo.toml + - crates/cyber-app/src/** + - crates/cyber-app/tests/support/scripted_model.rs + - crates/cyber-server/src/runtime/snapshot*.rs + - crates/cyber-server/src/runtime/rewind*.rs + - crates/cyber-server/src/http/sessions.rs + - crates/cyber-app/tests/snapshot_git_runtime.rs + - crates/cyber-server/src/workspace.rs - crates/cyber-core/src/binary/native/** - crates/cyber-core/src/binary/native.rs - Cargo.lock @@ -37,10 +45,19 @@ jobs: run: | set -euo pipefail cargo test --locked -p cyber-snapshot -- --nocapture 2>&1 | tee non-git-snapshot-proof.log + - name: Owned Git runtime acceptance candidate + shell: bash + run: | + set -euo pipefail + cargo test --locked -p cyber-snapshot --features native-git-acceptance -- --nocapture 2>&1 | tee git-adapter-proof.log + cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_git_runtime -- --nocapture 2>&1 | tee git-runtime-proof.log - name: Retain native evidence if: always() uses: actions/upload-artifact@v4 with: name: non-git-snapshot-proof-${{ matrix.os }} - path: non-git-snapshot-proof.log + path: | + non-git-snapshot-proof.log + git-adapter-proof.log + git-runtime-proof.log if-no-files-found: warn diff --git a/crates/cyber-app/Cargo.toml b/crates/cyber-app/Cargo.toml index c233fd80b..4148998cd 100644 --- a/crates/cyber-app/Cargo.toml +++ b/crates/cyber-app/Cargo.toml @@ -62,3 +62,6 @@ windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Syste [lints] workspace = true + +[features] +native-git-acceptance = ["cyber-snapshot/native-git-acceptance"] diff --git a/crates/cyber-app/tests/snapshot_git_runtime.rs b/crates/cyber-app/tests/snapshot_git_runtime.rs new file mode 100644 index 000000000..0f48c2fe8 --- /dev/null +++ b/crates/cyber-app/tests/snapshot_git_runtime.rs @@ -0,0 +1,308 @@ +#![cfg(all( + feature = "native-git-acceptance", + any(target_os = "linux", target_os = "macos") +))] +//! The real App/Store/worker path in an explicitly non-default acceptance build. +#[path = "support/scripted_model.rs"] +mod scripted_model; +use cyber_app::{App, AppOptions}; +use cyber_client::Client; +use cyber_core::paths::{DatabaseLocation, Paths}; +use cyber_server::runtime::SnapshotRestoreReceipt; +use serde_json::{Value, json}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +struct Fixture { + _temp: tempfile::TempDir, + app: App, + client: Client, + model: scripted_model::ScriptedModel, + paths: Paths, + project: PathBuf, + directory: PathBuf, + session: String, +} +impl Fixture { + async fn new(format: &str, scoped: bool) -> Self { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let paths = Paths { + data: root.join("data"), + config: root.join("config"), + state: root.join("state"), + cache: root.join("cache"), + tmp: root.join("tmp"), + }; + paths.ensure().unwrap(); + let project = root.join("project"); + git( + &root, + &[ + "init", + "--quiet", + &format!("--object-format={format}"), + project.to_str().unwrap(), + ], + ); + let directory = if scoped { + project.join("scope") + } else { + project.clone() + }; + std::fs::create_dir_all(&directory).unwrap(); + let model = scripted_model::ScriptedModel::start(|_| json!({"content":"Done."})); + let mut config = model.config(); + config["snapshots"] = json!({"max_file_bytes":4096}); + std::fs::write(paths.config.join("cyber.jsonc"), config.to_string()).unwrap(); + let app = build(&paths, &directory, &root).await; + let client = Client::embedded(app.embedded()).at(directory.to_str().unwrap()); + let session = client + .post( + "/sessions", + json!({"model":"test/main","mode":"accept-edits"}), + ) + .await + .unwrap()["data"]["id"] + .as_str() + .unwrap() + .to_owned(); + Self { + _temp: temp, + app, + client, + model, + paths, + project, + directory, + session, + } + } + async fn prompt(&self, text: &str, count: usize) -> String { + self.client + .post( + &format!("/sessions/{}/prompt", self.session), + json!({"parts":[{"type":"text","text":text}]}), + ) + .await + .unwrap(); + for _ in 0..1000 { + let messages = self.messages().await; + let messages = messages["data"].as_array().unwrap(); + if messages + .iter() + .filter(|message| message["kind"] == "assistant") + .count() + == count + && !self.app.runtime.is_running(&self.session) + { + return messages + .iter() + .rfind(|message| message["kind"] == "user") + .unwrap()["id"] + .as_str() + .unwrap() + .to_owned(); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("scripted prompt did not complete"); + } + async fn messages(&self) -> Value { + self.client + .get(&format!("/sessions/{}/messages?order=asc", self.session)) + .await + .unwrap() + } + fn receipt(&self) -> SnapshotRestoreReceipt { + let session = self.session.clone(); + self.app.store.read(move |db| { + let raw: String = db.query_row("SELECT json_extract(data,'$.snapshot_restore_receipt') FROM event WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_restore_receipt') IS NOT NULL ORDER BY rowid DESC LIMIT 1", [session], |row| row.get(0))?; + Ok(serde_json::from_str(&raw).unwrap()) + }).unwrap() + } + fn journal(&self, receipt: &SnapshotRestoreReceipt) -> PathBuf { + std::fs::read_dir(self.paths.data.join("snapshot/global")) + .unwrap() + .map(|entry| { + entry + .unwrap() + .path() + .join("journals") + .join(&receipt.origin.binding.journal_id) + }) + .find(|path| path.exists()) + .unwrap() + } +} +async fn build(paths: &Paths, directory: &Path, root: &Path) -> App { + App::build(AppOptions { + paths: paths.clone(), + home: root.join("home"), + database: DatabaseLocation::File(paths.data.join("cyber.db")), + default_directory: directory.into(), + sandbox_policy: None, + snapshots: true, + interactive: true, + password: None, + mcp_credentials: None, + remote_skill_fetcher: None, + workflow_cores: None, + isolation: None, + }) + .await + .unwrap() +} +fn git(directory: &Path, args: &[&str]) { + let output = std::process::Command::new("git") + .arg("-C") + .arg(directory) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} +fn image(value: u8) -> Vec { + let mut bytes = vec![value; 2 * 1024 * 1024 + 8192]; + bytes[0] = 0; + bytes +} + +#[tokio::test] +async fn actual_git_worker_commits_receipt_and_reconciles_without_source_replay() { + for (format, scoped) in [("sha1", false), ("sha1", true), ("sha256", true)] { + let f = Fixture::new(format, scoped).await; + let root_lock = cyber_server::workspace::mutation_lock(&f.project).unwrap(); + let scope_lock = cyber_server::workspace::mutation_lock(&f.directory).unwrap(); + assert!( + Arc::ptr_eq(&root_lock, &scope_lock), + "{format} scoped={scoped}" + ); + std::fs::write(f.directory.join("tracked"), image(b'b')).unwrap(); + std::fs::write(f.directory.join("owner"), vec![b'o'; 8192]).unwrap(); + std::os::unix::fs::symlink("tracked", f.directory.join("link")).unwrap(); + let tracked = if scoped { "scope/tracked" } else { "tracked" }; + git(&f.project, &["add", "--", tracked]); + let index = std::fs::read(f.project.join(".git/index")).unwrap(); + let user_config = std::fs::read(f.project.join(".git/config")).unwrap(); + let first = f.prompt("before", 1).await; + std::fs::write(f.directory.join("tracked"), image(b'a')).unwrap(); + std::fs::remove_file(f.directory.join("link")).unwrap(); + std::os::unix::fs::symlink("owner", f.directory.join("link")).unwrap(); + f.prompt("after", 2).await; + let mut request = json!({"message_id":first,"target":"code","action":"commit","expected_directory":f.directory}); + let preview = f + .client + .post( + &format!("/sessions/{}/rewind/preview", f.session), + request.clone(), + ) + .await + .unwrap(); + request["confirmation"] = preview["data"]["confirmation"].clone(); + f.client + .post(&format!("/sessions/{}/rewind/confirm", f.session), request) + .await + .unwrap(); + assert_eq!( + std::fs::read(f.directory.join("tracked")).unwrap(), + image(b'b') + ); + assert_eq!( + std::fs::read_link(f.directory.join("link")).unwrap(), + PathBuf::from("tracked") + ); + assert_eq!( + std::fs::read(f.directory.join("owner")).unwrap(), + vec![b'o'; 8192] + ); + assert_eq!(std::fs::read(f.project.join(".git/index")).unwrap(), index); + assert_eq!( + std::fs::read(f.project.join(".git/config")).unwrap(), + user_config + ); + let receipt = f.receipt(); + assert!(receipt.checkout.is_some()); + let journal = f.journal(&receipt); + let intent: Value = + serde_json::from_slice(&std::fs::read(journal.join("intent.json")).unwrap()).unwrap(); + assert_eq!(intent["backend"], "git"); + assert_eq!(intent["completed"], true); + assert_eq!(intent["acknowledged"], true); + assert_eq!(receipt.origin.binding.directory, f.directory); + reconcile_after_owner_edit(&f, &receipt).await; + f.app.runtime.shutdown().await; + } +} +struct Preserved { + messages: Value, + inference: usize, + outside: PathBuf, +} +async fn preserved_owner_edit(f: &Fixture) -> Preserved { + let messages = f.messages().await; + let inference = f.model.requests().len(); + let outside = f._temp.path().join("later-owner"); + std::fs::write(&outside, "later owner bytes").unwrap(); + std::fs::remove_file(f.directory.join("tracked")).unwrap(); + std::os::unix::fs::symlink(&outside, f.directory.join("tracked")).unwrap(); + Preserved { + messages, + inference, + outside, + } +} +async fn receipt_roundtrip(client: &Client, session: &str, receipt: &SnapshotRestoreReceipt) { + let endpoint = format!( + "/sessions/{session}/rewind/restores/{}", + receipt.origin.activity_id + ); + let lookup = client.get(&endpoint).await.unwrap(); + assert_eq!( + serde_json::from_value::(lookup["data"].clone()).unwrap(), + *receipt + ); + let acknowledgement = client + .post(&format!("{endpoint}/acknowledge"), json!({})) + .await + .unwrap(); + assert_eq!( + serde_json::from_value::(acknowledgement["data"].clone()).unwrap(), + *receipt + ); +} +async fn assert_preserved(f: &Fixture, client: &Client, state: &Preserved) { + assert_eq!( + std::fs::read_link(f.directory.join("tracked")).unwrap(), + state.outside + ); + assert_eq!(std::fs::read(&state.outside).unwrap(), b"later owner bytes"); + assert_eq!( + client + .get(&format!("/sessions/{}/messages?order=asc", f.session)) + .await + .unwrap(), + state.messages + ); + assert_eq!(f.model.requests().len(), state.inference); +} +async fn reconcile_after_owner_edit(f: &Fixture, receipt: &SnapshotRestoreReceipt) { + let state = preserved_owner_edit(f).await; + for _ in 0..2 { + receipt_roundtrip(&f.client, &f.session, receipt).await; + } + assert_preserved(f, &f.client, &state).await; + f.app.runtime.shutdown().await; + let reopened = build(&f.paths, &f.directory, f._temp.path()).await; + let client = Client::embedded(reopened.embedded()).at(f.directory.to_str().unwrap()); + receipt_roundtrip(&client, &f.session, receipt).await; + assert_preserved(f, &client, &state).await; + reopened.runtime.shutdown().await; +} diff --git a/crates/cyber-server/src/workspace.rs b/crates/cyber-server/src/workspace.rs index 5c6a5cf88..ff047a563 100644 --- a/crates/cyber-server/src/workspace.rs +++ b/crates/cyber-server/src/workspace.rs @@ -13,8 +13,18 @@ static LOCKS: OnceLock> = OnceLock::new(); /// Git Locations share their worktree root; non-Git Locations use their exact /// canonical directory. Callers acquire this before any individual path lock. pub fn mutation_lock(location: &Path) -> io::Result>> { - let root = cyber_core::project::worktree_root(location) - .unwrap_or_else(|| location.to_path_buf()) + let location = location.canonicalize()?; + // libgit2 may not discover newer Git repository formats. A marker can widen + // coordination to the worktree without granting repository/source authority. + // Native origin, Location and source proofs remain independently mandatory. + let root = cyber_core::project::worktree_root(&location) + .or_else(|| { + location + .ancestors() + .find(|path| path.join(".git").symlink_metadata().is_ok()) + .map(Path::to_path_buf) + }) + .unwrap_or(location) .canonicalize()?; let mut locks = LOCKS .get_or_init(Mutex::default) @@ -35,6 +45,32 @@ pub fn mutation_lock(location: &Path) -> io::Result>> { #[cfg(test)] mod tests { use super::*; + #[test] + fn sha256_scoped_locations_share_coordination_without_writing_project_identity() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let output = std::process::Command::new("git") + .args(["init", "--quiet", "--object-format=sha256"]) + .arg(&root) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let scope = root.join("scope/deep"); + std::fs::create_dir_all(&scope).unwrap(); + let first = mutation_lock(&root).unwrap(); + let second = mutation_lock(&scope).unwrap(); + assert!(Arc::ptr_eq(&first, &second)); + assert!(!root.join(".git/cyber-project-id").exists()); + let held = first.try_lock().unwrap(); + assert!(second.try_lock().is_err()); + drop(held); + assert!(second.try_lock().is_ok()); + } + #[tokio::test] async fn separate_owners_share_one_lock_until_the_owned_worker_releases_it() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/cyber-snapshot/Cargo.toml b/crates/cyber-snapshot/Cargo.toml index 6c824df90..443ab021c 100644 --- a/crates/cyber-snapshot/Cargo.toml +++ b/crates/cyber-snapshot/Cargo.toml @@ -37,3 +37,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "process", "time", "sync [lints] workspace = true + +[features] +# Exercise the real managed Git adapter before default admission acceptance. +native-git-acceptance = [] diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index ba852d6b4..b7464c905 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -935,10 +935,10 @@ async fn mixed_copy_git_images_keep_original_ids_scope_exclusions_and_new_journa .unwrap() .expected = ImageSource::Copy; assert!(mismatched.validate(&location, 6).is_err()); - assert!( - !repo - .snapshots - .owned_restore_available(location.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(location.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); assert!(!repo.root.join(".git/index").exists()); } @@ -1085,10 +1085,10 @@ async fn native_git_import_restores_files_modes_creations_deletions_and_retains_ assert!(journal.completed); assert!(journal.ownership.is_some()); assert_eq!(storage::read(&dir, "stage_2", 65536).unwrap(), b"after"); - assert!( - !repo - .snapshots - .owned_restore_available(repo.root.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(repo.root.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); let mut legacy_git = journal.clone(); legacy_git.version = 4; @@ -2140,10 +2140,10 @@ async fn native_git_raw_capture_supports_sha256_user_repositories_and_encoded_by let user_config = std::fs::read(repo.root.join(".git/config")).unwrap(); let target = repo.track(&repo.root).await; assert_eq!(target.len(), 64); - assert!( - !repo - .snapshots - .owned_restore_available(repo.root.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(repo.root.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); std::fs::write(repo.root.join("file.txt"), &after).unwrap(); let recorded = repo.track(&repo.root).await; @@ -2179,10 +2179,10 @@ async fn git_markers_refuse_non_git_admission_without_creating_identity_or_snaps std::fs::create_dir_all(&nested).unwrap(); for path in [&repo.root, &nested] { assert!(!repo.snapshots.non_git(path.to_str().unwrap())); - assert!( - !repo - .snapshots - .owned_restore_available(path.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(path.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); } assert!(!repo.root.join(".git/cyber-project-id").exists()); @@ -2353,10 +2353,10 @@ async fn sha256_linked_worktree_uses_its_common_repository_format_and_scoped_git common_index ); assert_eq!(std::fs::read(location.join("file")).unwrap(), b"before"); - assert!( - !repo - .snapshots - .owned_restore_available(location.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(location.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); } @@ -2530,10 +2530,10 @@ async fn native_git_preview_and_path_selection_include_normalization_only_change shadow.changed_native(&target, &recorded).await.unwrap(), ["file.txt"] ); - assert!( - !repo - .snapshots - .owned_restore_available(repo.root.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(repo.root.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); let preview = repo .snapshots @@ -2631,10 +2631,10 @@ async fn git_lifetime_port_retains_exact_owner_lock_without_enabling_source_admi .is_err() ); assert!(repo.snapshots.lock_worker(binding.clone()).await.is_err()); - assert!( - !repo - .snapshots - .owned_restore_available(repo.root.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(repo.root.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); drop(lifetime); let lifetime = repo.snapshots.lock_worker(binding.clone()).await.unwrap(); @@ -2654,10 +2654,10 @@ async fn git_lifetime_port_retains_exact_owner_lock_without_enabling_source_admi assert_eq!(lifetime.binding(), &binding); assert_eq!(lifetime.checkout_bindings().unwrap(), Some(vec![])); assert!(repo.store(&repo.root).recovery().unwrap().is_empty()); - assert!( - !repo - .snapshots - .owned_restore_available(repo.root.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(repo.root.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); } @@ -2686,10 +2686,10 @@ async fn git_lifetime_port_keeps_scoped_locations_and_managed_checkout_completen assert_eq!(lifetime.checkout_bindings().unwrap(), None); assert!(repo.store(&scope).recovery().unwrap().is_empty()); assert_eq!(std::fs::read(scope.join("owner")).unwrap(), b"owner bytes"); - assert!( - !repo - .snapshots - .owned_restore_available(scope.to_str().unwrap()) + assert_eq!( + repo.snapshots + .owned_restore_available(scope.to_str().unwrap()), + cfg!(feature = "native-git-acceptance") ); } diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index d0bcc2304..895870c3d 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -252,7 +252,7 @@ impl GitSnapshots { let shadow = self .shadow(&request.directory) .ok_or_else(|| RestoreError::Failed("Git snapshots are unavailable".into()))?; - let options = self.copy_options(&request.directory); + let options = self.git_options(&request.directory); let lock = cyber_server::workspace::mutation_lock(&shadow.worktree) .map_err(|error| RestoreError::Failed(error.to_string()))?; let guard = lock.lock_owned().await; @@ -290,6 +290,14 @@ impl GitSnapshots { } } #[cfg(any(target_os = "macos", target_os = "linux"))] + fn git_options(&self, directory: &str) -> copy::Options { + let mut options = self.copy_options(directory); + // The Git configured file threshold omits only untracked files. Tracked + // images retain the raw Git reader's separate bound and total budget. + options.max_file = options.max_total.min(64 * 1024 * 1024); + options + } + #[cfg(any(target_os = "macos", target_os = "linux"))] fn recovery_options(&self, directory: &str) -> copy::Options { let mut options = self.copy_options(directory); // Retained files remain inspectable after capture limits are lowered or @@ -657,7 +665,16 @@ impl Snapshots for GitSnapshots { }) } fn owned_restore_available(&self, directory: &str) -> bool { - cfg!(any(target_os = "macos", target_os = "linux")) && self.non_git(directory) + cfg!(any(target_os = "macos", target_os = "linux")) + && (self.non_git(directory) + || (cfg!(feature = "native-git-acceptance") + && (self.config)(Path::new(directory)).get("snapshots") + != Some(&Value::Bool(false)) + && !self + .private_roots + .iter() + .any(|root| Path::new(directory).starts_with(root)) + && git::repository_marker(Path::new(directory)))) } fn restore_owned( @@ -675,8 +692,8 @@ impl Snapshots for GitSnapshots { } #[cfg(any(target_os = "macos", target_os = "linux"))] { - // Keep the same admission gate for direct port callers. The Git - // journal bridge cannot be activated before its remaining gates. + // Direct callers share the runtime availability gate. Candidate + // Git admission is confined to the explicit acceptance build. if !self.owned_restore_available(&request.directory) { return Err(RestoreError::Failed( "Owned snapshot restore is unavailable for this Location".into(), @@ -693,17 +710,10 @@ impl Snapshots for GitSnapshots { ) })?) }; - let config = (self.config)(Path::new(&request.directory)); - let (ignore, max_file) = Self::options_value(&config); - let max_total = config - .pointer("/snapshots/fallback_max_bytes") - .and_then(Value::as_u64) - .unwrap_or(200 * 1024 * 1024); - let options = copy::Options { - ignore, - max_file: max_file.min(max_total).min(64 * 1024 * 1024), - max_total, - private_roots: self.private_roots.clone(), + let options = if shadow.is_some() { + self.git_options(&request.directory) + } else { + self.copy_options(&request.directory) }; let prepared = match shadow { Some(shadow) => Some( diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index 316d267d7..a26164a9d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3646,3 +3646,78 @@ intentionally excluded from direct top-level selection. Windows-only/doc binarie again selected zero. Final all-target Clippy, format/diff, strict OpenSpec (117), lint (0/25) and SDK generation consistency passed. No further scope or acceptance claim changes: source admission and parent tasks remain open as stated above. + +### Actual Git runtime acceptance candidate (2026-11-07) + +The adapter already contained a Git branch in restore_owned, but its availability +gate made it unreachable. A non-default native-git-acceptance Cargo feature now +exercises that same production branch through the real App, runtime-issued workers, +Store settlement and acknowledgement. It is forwarded by cyber-app for acceptance +invocations. Normal builds and the release workflow (which does not select all +features) retain the existing Git gate. The marker-only candidate availability +check creates no project identity cache and cannot substitute for later Location, +source/image, lifecycle, original-worker or checkout validation. Private roots and +disabled snapshots still refuse. No public API/schema or force-clear path is added. + +Owned Git preview and restore now use a distinct bounded raw-image limit (up to +64 MiB and the combined budget) rather than applying max_file_bytes to tracked +files. That setting continues to govern Git untracked omission. Non-Git options +retain their old limit. The new App fixture uses a binary tracked image above +2 MiB with an explicit 4 KiB omission threshold, a link and an independently omitted +8 KiB owner file. Real prompt capture, rewind preview/confirm and native Store +receipts restore the tracked bytes/link while preserving that omitted file and the +user index/config. The receipt retains complete versioned lifetime/checkout proof; +these cases use the complete empty resource set, not managed-checkout acceptance. + +The first App invocation reached both SHA-1 root/scoped restore/receipt/reconcile +cases, then failed its root-vs-scope lock assertion for SHA-256. Workspace +coordination used only libgit2 discovery and therefore chose different domains when +that format was unavailable to libgit2. A read-only ancestor .git marker fallback +now conservatively widens the process-local lock domain. It does not grant Git +validity, permission or source authority and does not run Git or create identity +caches. A dedicated default-build regression verifies one shared live lock for a +SHA-256 root and scope and no cyber-project-id creation. The subsequent first full +App invocation passed its three cases in 55.81 seconds; the later reopened-App +extension and final results follow below. + +The runtime fixture replaces a restored file with an absolute symlink to later +owner bytes, repeats committed lookup/explicit acknowledgement, and compares +messages and inference counts. It then shuts down the actual runtime, constructs +a fresh App over the same Store and repeats only lookup/metadata reconciliation. +This is an App shutdown/reopen, not an OS process-death cut or lost writer-reply +injection. Existing immutable receipt and native ownership gates remain in force. +The native CI workflow now schedules both default snapshot proof and separate +candidate-adapter/App invocations on Linux x64/ARM and macOS Intel/ARM and retains +separate logs. That workflow edit is not evidence that those jobs ran. + +Clippy caught an inefficient double-ended iterator and a 24/15 test helper; +receipt roundtrip, source preservation and reopened-runtime responsibilities were +split without lint allowances. Code-review skill inspection checked default-vs- +candidate behavior, release feature selection, no metadata writes from availability +or coordination, backend image limits, complete resource proof, exact receipt +consumption and no replay after later owner edits. Managed worktrees in the default +application data layout still need resource-verified storage-root integration: +copy::Store currently rejects Locations under its data root, and the shadow's broad +data exclusion also needs reconciliation for those legitimate source trees. The +new empty-set cases do not prove that support. Keep/rollback after actual Git +partial replacement, all native cuts, cleanup of managed/Git payloads, complete +configuration/filter/limit compatibility and source-bound platform acceptance +remain required. No M2–M8 or parent task is accepted by this candidate increment. + +Final local macOS arm64 checks: the reopened-App acceptance test passed all three +parameter cases in 47.32 seconds. The default snapshot package passed 138 library, +18 file-copy and 22 Git integration cases; the candidate package passed the same +138/18/22 cases. Their library times were 60.47 and 87.90 seconds, respectively. +Both modes intentionally exclude six library and one integration subprocess +fixtures from top-level selection. Windows-only/doc binaries select zero on this +host. All three workspace tests passed in 0.04 seconds, including the dedicated +SHA-256 regression, and all 46 snapshot-worker regressions passed in 4.70 seconds. +A later lint-only user-message iterator change uses rfind; the App execution above +preceded that equivalent helper change. Final candidate App/server/snapshot +all-target Clippy, format/diff, SDK generation consistency, strict OpenSpec (117) +and spec lint (0 errors/25 warnings) passed. Workflow YAML syntax was checked with +Ruby's YAML library after Python PyYAML was unavailable. Every Cargo test/Clippy +invocation used CARGO_INCREMENTAL=0 and --locked. C sandbox deprecations and a linker +unwind warning were observed without unchanged-main attribution. No full workspace, +CLI/TUI acceptance rerun, SDK execution or source-bound CI/other-platform run was +performed. No push/PR/merge, default activation or parent/issue completion is claimed. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 88c3a6147..79c5facc4 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -934,3 +934,27 @@ the native cancellation event. Version-3 cleanup MUST match that exact committed digest. Missing historical digest evidence cannot be inferred from current native files and grants no cleanup authority; legacy lookup/metadata acknowledgement retains its existing behavior. This adds no public receipt field or source replay. + + +### Runtime Git acceptance build + +A non-default native-git-acceptance Cargo feature MAY exercise the existing owned +Git adapter through actual App/runtime workers before normal admission is enabled. +The candidate SHALL use the same reservation, complete resource publication, +retained synchronous worker, consuming Store pair and metadata acknowledgement as +ordinary file-copy restore. It SHALL NOT bypass legacy/unknown ownership fences, +worker identity or acknowledgement checks. Default builds SHALL keep the existing +Git admission gate until full integration/compatibility/platform acceptance. +Feature-build results SHALL be identified separately and SHALL NOT close that gate +or parent tasks by themselves. Git preparation/preview SHALL use its bounded raw +image read limit for tracked images; snapshots.max_file_bytes SHALL retain its +existing role as the Git untracked-file omission threshold. Non-Git file limits +and combined image/storage caps SHALL remain enforced. + + +Workspace coordination SHALL remain shared by root and scoped Git Locations even +when libgit2 cannot discover the repository format. A read-only ancestor-marker +fallback MAY conservatively widen only the process-local mutation-lock domain; +it SHALL NOT create identity caches or grant Git validity, native source, +permission, recovery or acknowledgement authority. Native source/Location and +exact worker ownership checks remain independent and mandatory. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 21ea76858..a47eafcc9 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -432,3 +432,22 @@ - **GIVEN** captured file images, effective configuration, ignores and size-excluded names are unchanged - **WHEN** harmless empty directories add only absent source .gitignore observations - **THEN** those absence checks SHALL remain available during verification without changing serialized classification provenance or checkpoint identity + + +#### Scenario: Explicit owned Git runtime acceptance build +- **GIVEN** a non-default native-git-acceptance build exercises the Git adapter +- **WHEN** a code rewind uses actual App/runtime workers and the consuming Store +- **THEN** ordinary restore SHALL retain exact origin/lifetime/resource proof, commit the native/Session pair and publish acknowledgement through the existing guarded path +- **AND** committed lookup and explicit metadata reconciliation, including a reopened App, SHALL preserve later owner files/symlinks, messages and inference counts without source replay +- **AND** this acceptance feature SHALL NOT enable default Git admission or waive incomplete compatibility, managed-resource, cleanup, process-cut or platform acceptance + +#### Scenario: Tracked Git images exceed the untracked omission threshold +- **GIVEN** a tracked Git image exceeds snapshots.max_file_bytes but fits the bounded raw-image reader and combined budget +- **WHEN** the native candidate previews or restores it +- **THEN** the untracked omission threshold SHALL NOT reject that tracked image, and independently omitted untracked files SHALL remain preserved +- **AND** non-Git file limits and combined image/storage caps SHALL remain enforced + +#### Scenario: Scoped SHA-256 worktree mutation coordination +- **WHEN** root and scoped Locations identify a Git repository format unavailable to libgit2 discovery +- **THEN** read-only ancestor marker discovery SHALL keep them in one conservative process-local mutation-lock domain without running Git or writing project identity caches +- **AND** that lock domain SHALL NOT grant source, permission, recovery, acknowledgement or Git-validity authority diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 701193a6e..a20138651 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -587,3 +587,16 @@ Supported capture/preparation/restore and mutation-during-classification positiv and denials are covered. Complete configuration/ABA/filter/limit compatibility, managed Git runtime/checkout/recovery/cleanup and source-bound platform acceptance remain open; managed source admission and all parent acceptance stay disabled. + +Runtime Git candidate continuation: add a non-default native-git-acceptance build +that takes the same owned restore/preview/changed-path ports and actual runtime +workers, without changing default admission. Git tracked-image reads use the raw +reader/total bound while configured max_file_bytes remains the untracked omission +threshold. Actual App cases cover root/scoped SHA-1 and scoped SHA-256, binary +tracked images above that threshold, links, Store receipts, native acknowledgement, +lookup/reconciliation and later owner edits. Root/scoped SHA-256 coordination now +shares a read-only ancestor-marker lock domain when libgit2 cannot discover it; +this is coordination, never source/permission authority. Add candidate jobs to the +native matrix, keeping their evidence distinct. Default admission, complete managed +resource/storage-root integration, keep/rollback/process-death/runtime cuts, +compatibility/reclamation and full platform acceptance remain open with M2–M8. From a6f80a01aebdb0fae05fba6990398a749cad89da Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Sat, 7 Nov 2026 14:33:27 -0400 Subject: [PATCH 49/87] Restore registered managed checkout sources with owned workers --- .../cyber-app/tests/snapshot_git_runtime.rs | 113 +++++++++++++++++- .../src/runtime/rewind_preview.rs | 10 +- .../src/runtime/snapshot_recovery.rs | 5 +- .../cyber-server/src/runtime/snapshot_work.rs | 46 +++++++ .../src/runtime/snapshot_work_tests.rs | 28 +++++ crates/cyber-snapshot/src/copy/links.rs | 2 +- .../cyber-snapshot/src/copy/managed_tests.rs | 106 ++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 19 ++- crates/cyber-snapshot/src/copy/restore.rs | 6 + crates/cyber-snapshot/src/copy/tree.rs | 4 +- crates/cyber-snapshot/src/git.rs | 35 +++++- docs/goal-delivery.md | 10 ++ .../implementation.md | 61 ++++++++++ .../managed-recovery.md | 11 ++ .../specs/snapshots-checkpoints/spec.md | 12 ++ .../complete-daily-coding-surfaces/tasks.md | 13 ++ 16 files changed, 470 insertions(+), 11 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/managed_tests.rs diff --git a/crates/cyber-app/tests/snapshot_git_runtime.rs b/crates/cyber-app/tests/snapshot_git_runtime.rs index 0f48c2fe8..7030bf9fc 100644 --- a/crates/cyber-app/tests/snapshot_git_runtime.rs +++ b/crates/cyber-app/tests/snapshot_git_runtime.rs @@ -156,7 +156,7 @@ async fn build(paths: &Paths, directory: &Path, root: &Path) -> App { .await .unwrap() } -fn git(directory: &Path, args: &[&str]) { +fn git(directory: &Path, args: &[&str]) -> String { let output = std::process::Command::new("git") .arg("-C") .arg(directory) @@ -168,6 +168,7 @@ fn git(directory: &Path, args: &[&str]) { "{}", String::from_utf8_lossy(&output.stderr) ); + String::from_utf8(output.stdout).unwrap().trim().to_owned() } fn image(value: u8) -> Vec { let mut bytes = vec![value; 2 * 1024 * 1024 + 8192]; @@ -175,6 +176,116 @@ fn image(value: u8) -> Vec { bytes } +#[tokio::test] +async fn managed_worktree_in_default_data_directory_restores_through_owned_worker() { + let mut f = Fixture::new("sha1", false).await; + std::fs::write(f.project.join("tracked"), "initial").unwrap(); + git(&f.project, &["add", "tracked"]); + git( + &f.project, + &[ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "-c", + "commit.gpgsign=false", + "commit", + "--quiet", + "-m", + "Initial", + ], + ); + let created = f + .client + .post( + "/worktrees", + json!({"name":"managed", "session":{"model":"test/main"}}), + ) + .await + .unwrap(); + f.directory = PathBuf::from(created["data"]["worktree"]["path"].as_str().unwrap()); + f.session = created["data"]["session"]["id"] + .as_str() + .unwrap() + .to_owned(); + f.client = Client::embedded(f.app.embedded()).at(f.directory.to_str().unwrap()); + assert!(f.directory.starts_with(f.paths.data.join("worktrees"))); + let index_path = PathBuf::from(git( + &f.directory, + &["rev-parse", "--path-format=absolute", "--git-path", "index"], + )); + let index = std::fs::read(&index_path).unwrap(); + let config = std::fs::read(f.project.join(".git/config")).unwrap(); + std::fs::write(f.directory.join("tracked"), "before").unwrap(); + let first = f.prompt("before", 1).await; + std::fs::write(f.directory.join("tracked"), "after").unwrap(); + f.prompt("after", 2).await; + let mut request = json!({"message_id":first,"target":"code","action":"commit","expected_directory":f.directory}); + let preview = f + .client + .post( + &format!("/sessions/{}/rewind/preview", f.session), + request.clone(), + ) + .await + .unwrap(); + request["confirmation"] = preview["data"]["confirmation"].clone(); + tokio::time::timeout( + Duration::from_secs(30), + f.client + .post(&format!("/sessions/{}/rewind/confirm", f.session), request), + ) + .await + .expect("managed confirmation timed out") + .unwrap(); + assert_eq!( + std::fs::read_to_string(f.directory.join("tracked")).unwrap(), + "before" + ); + let receipt = f.receipt(); + assert_eq!( + serde_json::to_value(&receipt.origin).unwrap()["worktree_id"], + created["data"]["worktree"]["id"] + ); + let checkout = receipt.checkout.as_ref().unwrap(); + let journal = f.journal(&receipt); + let phase: Value = serde_json::from_slice( + &std::fs::read( + journal + .parent() + .unwrap() + .parent() + .unwrap() + .join("workers") + .join(checkout.lifetime.source_phase.as_ref().unwrap()), + ) + .unwrap(), + ) + .unwrap(); + let bindings: Vec = + serde_json::from_value(phase["checkout_bindings"].clone()).unwrap(); + assert_eq!(bindings.len(), 1); + assert_eq!( + bindings[0].worktree_id, + receipt.origin.worktree_id.as_deref().unwrap() + ); + assert_eq!( + checkout + .lifetime + .checkout_resources_digest(&bindings) + .unwrap(), + checkout.resources_digest + ); + assert_eq!(std::fs::read(&index_path).unwrap(), index); + assert_eq!( + std::fs::read(f.project.join(".git/config")).unwrap(), + config + ); + reconcile_after_owner_edit(&f, &receipt).await; + f.app.runtime.shutdown().await; +} + #[tokio::test] async fn actual_git_worker_commits_receipt_and_reconciles_without_source_replay() { for (format, scoped) in [("sha1", false), ("sha1", true), ("sha256", true)] { diff --git a/crates/cyber-server/src/runtime/rewind_preview.rs b/crates/cyber-server/src/runtime/rewind_preview.rs index 3ca2a29c8..142902ea8 100644 --- a/crates/cyber-server/src/runtime/rewind_preview.rs +++ b/crates/cyber-server/src/runtime/rewind_preview.rs @@ -352,8 +352,8 @@ impl Runtime { .await } - /// Revalidate before any restore. Hold the Session projection through the owned - /// restore and event commit so a new prompt cannot change its branch mid-effect. + /// Revalidate before restore. Owned workers fence Session revisions and retain + /// their resource claims through filesystem work and the consuming event commit. pub async fn rewind_confirm( &self, session_id: &str, @@ -414,8 +414,12 @@ impl Runtime { expected: code.baseline.clone(), paths, }; - let worker = self.snapshot_worker(handle, &state).await?; + // Native checkout admission resolves policy through the runtime and + // may read this Session. Keep an immutable revision, not its mutex, + // across that host call; the worker fences stale revisions. + let captured = state.clone(); drop(state); + let worker = self.snapshot_worker(handle, &captured).await?; let completed = self .inner .options diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index df2d923c9..70e9a11ac 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -82,7 +82,9 @@ impl Runtime { .origin .ok_or_else(|| RuntimeError::Conflict("Recovery lacks exact native origin".into()))?; let handle = self.inner.handle(session).await?; - let state = handle.state.lock().await; + // Managed native admission may resolve policy through this Session. + // The captured revision is fenced by the worker before source effects. + let state = handle.state.lock().await.clone(); let directory = state.info.directory.clone(); let worker = self .snapshot_recovery_worker( @@ -95,7 +97,6 @@ impl Runtime { }, ) .await?; - drop(state); let completed = self .inner .options diff --git a/crates/cyber-server/src/runtime/snapshot_work.rs b/crates/cyber-server/src/runtime/snapshot_work.rs index 33b349430..3ba929054 100644 --- a/crates/cyber-server/src/runtime/snapshot_work.rs +++ b/crates/cyber-server/src/runtime/snapshot_work.rs @@ -235,6 +235,42 @@ impl SnapshotWork<'_> { Ok(()) } + /// Source admission for a registered checkout within private application + /// data. Identification alone cannot grant this: verify the actual worker's + /// complete retained native claims on both sides of the registry read. + pub fn authorize_managed_location(&self, directory: &std::path::Path) -> Result<(), String> { + self.authorize()?; + if directory != self.owner.directory { + return Err("Managed source differs from snapshot worker Location".into()); + } + let managed = cyber_core::worktrees::Repository::managed_locations_at(directory) + .map_err(|error| error.to_string())?; + let bindings = self.owner.location()?.bindings()?; + if managed.is_empty() || managed.len() != bindings.len() { + return Err("Managed source requires the complete native checkout set".into()); + } + for (_, managed) in managed { + if !managed.ready + || !bindings.iter().any(|binding| { + binding.version == 2 + && binding.worktree_id == managed.id + && binding.name == managed.name + && binding.common_dir == managed.common_dir + && binding.owner_id == self.owner.scope.id() + && binding.lifetime.as_ref().is_some_and(|owner| { + owner.directory == directory + && owner.activity_id == self.owner.scope.id() + && owner.session_id == self.owner.scope.control.source + && owner.session_created_id == self.owner.created_id + }) + }) + { + return Err("Managed source differs from retained native checkout proof".into()); + } + } + self.authorize() + } + /// Keep Session changes outside the exact synchronous namespace mutation. /// The longer-running worker keeps a separate admission owner so Stop and /// read-only inspection need not wait on a held Session projection mutex. @@ -576,6 +612,11 @@ impl Runtime { let admission = SnapshotAdmission(handle.clone()); let authority = super::snapshot_recovery_authority::RecoveryAuthority::capture(self, &state.info.id)?; + if self.inner.store.aggregate_seq(&state.info.id)? != Some(state.last_seq) { + return Err(RuntimeError::Conflict( + "Recovery Session changed before worker admission".into(), + )); + } request.authority = Some(authority.clone()); let id = super::snapshot_recovery::request_owner( &request.original.session_created_id, @@ -710,6 +751,11 @@ impl Runtime { .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) .map_err(|_| RuntimeError::Busy(state.info.id.clone()))?; let admission = SnapshotAdmission(handle.clone()); + if self.inner.store.aggregate_seq(&state.info.id)? != Some(state.last_seq) { + return Err(RuntimeError::Conflict( + "Snapshot Session changed before worker admission".into(), + )); + } let id = cyber_core::ids::new_id("op"); let lifetime = self.snapshot_lifetime(state, &id).await?; let scope = activity::Scope::reserve_snapshot_owned( diff --git a/crates/cyber-server/src/runtime/snapshot_work_tests.rs b/crates/cyber-server/src/runtime/snapshot_work_tests.rs index 2691788eb..24bcd3cc9 100644 --- a/crates/cyber-server/src/runtime/snapshot_work_tests.rs +++ b/crates/cyber-server/src/runtime/snapshot_work_tests.rs @@ -2248,6 +2248,34 @@ async fn writer_snapshot_fence_rejects_input_even_when_the_callers_precheck_is_s runtime.shutdown().await; } +#[tokio::test] +async fn captured_snapshot_revision_refuses_changes_before_native_reservation() { + let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; + let handle = runtime.inner.handle(&session).await.unwrap(); + let captured = handle.state.lock().await.clone(); + runtime + .rename(&session, "owner changed Session") + .await + .unwrap(); + let count = || { + runtime + .inner + .store + .read(|db| { + Ok(db.query_row("SELECT count(*) FROM event", [], |row| row.get::<_, i64>(0))?) + }) + .unwrap() + }; + let before = count(); + let result = runtime.snapshot_worker(&handle, &captured).await; + assert!( + matches!(result, Err(RuntimeError::Conflict(message)) if message.contains("changed before worker admission")) + ); + assert!(!handle.snapshot_active.load(Ordering::SeqCst)); + assert_eq!(count(), before); + runtime.shutdown().await; +} + #[tokio::test] async fn ordinary_snapshot_reservation_cannot_race_a_pending_generic_native_owner() { let (_temp, runtime, session, _, _) = super::super::binary_documents_tests::fixture().await; diff --git a/crates/cyber-snapshot/src/copy/links.rs b/crates/cyber-snapshot/src/copy/links.rs index 831b483f3..9bc8d126b 100644 --- a/crates/cyber-snapshot/src/copy/links.rs +++ b/crates/cyber-snapshot/src/copy/links.rs @@ -266,7 +266,7 @@ impl Store { .strip_prefix(boundary) .map_err(|_| "Symbolic-link target is outside the worktree")?; if super::tree::protected(relative) - || target.starts_with(&self.data_path) + || self.private_data_path(target) || self .options .private_roots diff --git a/crates/cyber-snapshot/src/copy/managed_tests.rs b/crates/cyber-snapshot/src/copy/managed_tests.rs new file mode 100644 index 000000000..7128e4120 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/managed_tests.rs @@ -0,0 +1,106 @@ +//! Registration permits observation, never unowned source mutation. +use super::*; + +fn git(root: &Path, args: &[&str]) { + let output = std::process::Command::new("git") + .arg("-C") + .arg(root) + .args(args) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); +} + +#[test] +fn registered_data_source_requires_worker_and_keeps_private_boundaries() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let repo = root.join("repo"); + let data = root.join("data"); + std::fs::create_dir(&repo).unwrap(); + std::fs::create_dir(&data).unwrap(); + git(&repo, &["init", "--quiet"]); + git( + &repo, + &[ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "--quiet", + "-m", + "Initial", + ], + ); + let source = data.join("worktrees/managed"); + git( + &repo, + &[ + "worktree", + "add", + "--quiet", + "-b", + "managed", + source.to_str().unwrap(), + ], + ); + let options = Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + }; + assert!(Store::open(&data, source.to_str().unwrap(), options.clone()).is_err()); + let managed = cyber_core::worktrees::Managed { + id: "wt_fixture".into(), + name: "managed".into(), + path: source.clone(), + branch: "managed".into(), + base: "master".into(), + common_dir: repo.join(".git"), + ready: true, + included: vec![], + }; + let records = repo.join(".git/cyber-worktrees"); + std::fs::create_dir(&records).unwrap(); + let record = records.join("managed.json"); + std::fs::write(&record, serde_json::to_vec(&managed).unwrap()).unwrap(); + let mut protected_options = options.clone(); + protected_options.private_roots.push(source.clone()); + assert!(Store::open(&data, source.to_str().unwrap(), protected_options).is_err()); + let store = Store::open(&data, source.to_str().unwrap(), options.clone()).unwrap(); + std::fs::write(source.join("tracked"), "owner").unwrap(); + assert!(store.capture().unwrap().files.contains_key("tracked")); + assert!(!store.private_data_path(&source.join("tracked"))); + assert!(store.private_data_path(&data.join("cyber.db"))); + assert!(store.private_data_path(&data.join("snapshot/global"))); + assert!(store.check_link_target("link", b"tracked", &source).is_ok()); + assert!( + store + .check_link_target("link", b"../../cyber.db", &source) + .is_err() + ); + let error = store + .prepare_journal(vec![], "restore_fixture", None, None, None) + .unwrap_err(); + assert!(error.contains("owned snapshot worker"), "{error}"); + assert_eq!( + std::fs::read_to_string(source.join("tracked")).unwrap(), + "owner" + ); + assert!( + storage::names(&store.journals, ENTRY_LIMIT) + .unwrap() + .is_empty() + ); + let mut changed = managed; + changed.ready = false; + std::fs::write(&record, serde_json::to_vec(&changed).unwrap()).unwrap(); + assert!(store.verify_root().is_err()); + drop(store); + assert!(Store::open(&data, source.to_str().unwrap(), options).is_err()); +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 0c69a9ed5..32c92ae3d 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -10,6 +10,8 @@ mod git_images; pub(crate) mod git_rules; mod inventory; mod links; +#[cfg(test)] +mod managed_tests; mod ownership; mod preparation; pub(crate) mod reclamation; @@ -76,6 +78,7 @@ pub(crate) struct Store { root: PathBuf, root_id: String, data_path: PathBuf, + managed_source: Option, work: Dir, namespace: Dir, @@ -180,7 +183,8 @@ impl Store { std::fs::create_dir_all(data).map_err(fail)?; } let data_path = data.canonicalize().map_err(fail)?; - if root.starts_with(&data_path) { + let managed_source = crate::git::managed_data_source(&data_path, &root)?; + if root.starts_with(&data_path) && managed_source.is_none() { return Err("Private owner state cannot be a snapshot Location".into()); } let work = Dir::open_ambient_dir(&root, cap_std::ambient_authority()).map_err(fail)?; @@ -224,6 +228,7 @@ impl Store { root, root_id, data_path, + managed_source, work, namespace: dir, objects, @@ -235,6 +240,11 @@ impl Store { }) } fn verify_root(&self) -> Result<()> { + if self.managed_source.is_some() + && crate::git::managed_data_source(&self.data_path, &self.root)? != self.managed_source + { + return Err("Managed snapshot Location registration changed".into()); + } if !std::fs::symlink_metadata(&self.root) .map_err(fail)? .file_type() @@ -249,6 +259,13 @@ impl Store { } Ok(()) } + fn private_data_path(&self, path: &Path) -> bool { + path.starts_with(&self.data_path) + && !self.managed_source.as_ref().is_some_and(|managed| { + path.starts_with(&managed.path) + && !path.starts_with(self.data_path.join("snapshot")) + }) + } pub(super) fn lock_identity(&self) -> Result { storage::identity( self._lock diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 7026f1f5e..21023fb26 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -530,6 +530,12 @@ impl Store { context: Option, mut resolution: Option, ) -> Result { + if self.managed_source.is_some() { + self.verify_root()?; + worker + .ok_or("Managed private-data source requires an owned snapshot worker")? + .authorize_managed_location(&self.root)?; + } let origin = worker .map(|worker| { worker.authorize()?; diff --git a/crates/cyber-snapshot/src/copy/tree.rs b/crates/cyber-snapshot/src/copy/tree.rs index 6c27670b1..cbc306f4e 100644 --- a/crates/cyber-snapshot/src/copy/tree.rs +++ b/crates/cyber-snapshot/src/copy/tree.rs @@ -87,7 +87,7 @@ impl Store { return Err("Snapshot path limit exceeded".into()); } let metadata = dir.symlink_metadata(item.file_name()).map_err(fail)?; - let excluded = self.root.join(&path).starts_with(&self.data_path) + let excluded = self.private_data_path(&self.root.join(&path)) || protected(&path) || self .options @@ -210,7 +210,7 @@ impl Store { pub(super) fn check_source_path(&self, path: &str) -> Result<()> { safe_path(path)?; if protected(Path::new(path)) - || self.root.join(path).starts_with(&self.data_path) + || self.private_data_path(&self.root.join(path)) || self .options .private_roots diff --git a/crates/cyber-snapshot/src/git.rs b/crates/cyber-snapshot/src/git.rs index b44103459..418dc01cc 100644 --- a/crates/cyber-snapshot/src/git.rs +++ b/crates/cyber-snapshot/src/git.rs @@ -99,6 +99,30 @@ pub(crate) fn repository_marker(location: &Path) -> bool { }) } +/// Identification permits reads and private metadata reservation only. The +/// native journal additionally requires retained runtime checkout authority. +pub(crate) fn managed_data_source( + data: &Path, + location: &Path, +) -> Result, String> { + if !location.starts_with(data) { + return Ok(None); + } + if location == data || location.starts_with(data.join("snapshot")) { + return Ok(None); + } + let managed = cyber_core::worktrees::Repository::managed_at(location) + .map_err(|error| error.to_string())? + .map(|(_, managed)| managed); + Ok(managed.filter(|managed| { + managed.ready + && !managed.id.is_empty() + && managed.path.starts_with(data) + && managed.path != data + && !managed.path.starts_with(data.join("snapshot")) + })) +} + impl Shadow { pub fn locate(root: &Path, location: &Path) -> Option { let project = cyber_core::project::identify(location); @@ -120,7 +144,16 @@ impl Shadow { common_dir: git_path(&common_dir), pathspec: if rel.is_empty() { ".".into() } else { rel }, catalogs: Arc::new(Mutex::new(catalog::CatalogCache::new())), - private_roots: vec![root.parent().unwrap_or(root).to_path_buf()], + private_roots: vec![ + if managed_data_source(root.parent().unwrap_or(root), &location) + .ok()? + .is_some() + { + root.to_path_buf() + } else { + root.parent().unwrap_or(root).to_path_buf() + }, + ], #[cfg(any(target_os = "macos", target_os = "linux"))] capture_settings: prepared::CaptureSettings { ignore: vec![], diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 0ff731bbe..5c0f7ea22 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -217,3 +217,13 @@ unknown fences remain protected. This is local implementation progress, not full recovery delivery: actual managed Git source callbacks, partial construction, reference-aware reclamation and complete native/platform/client acceptance remain open. See the daily coding implementation evidence and authoritative parent tasks. + +The non-default Git candidate now exercises an actual ordinary owned restore in a +managed checkout created under the default application data directory. Source +observation is registration-bound; source intents require the runtime worker's +complete retained native checkout proof. Admission releases the Session mutex for +host policy lookup and fences captured revisions. The App verifies persisted +resources, unchanged user Git metadata and reopened acknowledgement without source +replay. Default Git admission stays off. Managed partial replacement/recovery, +construction/reclamation and complete native/platform acceptance remain required; +this positive does not complete recovery or either issue. diff --git a/openspec/changes/complete-daily-coding-surfaces/implementation.md b/openspec/changes/complete-daily-coding-surfaces/implementation.md index a26164a9d..46d349dc8 100644 --- a/openspec/changes/complete-daily-coding-surfaces/implementation.md +++ b/openspec/changes/complete-daily-coding-surfaces/implementation.md @@ -3721,3 +3721,64 @@ invocation used CARGO_INCREMENTAL=0 and --locked. C sandbox deprecations and a l unwind warning were observed without unchanged-main attribution. No full workspace, CLI/TUI acceptance rerun, SDK execution or source-bound CI/other-platform run was performed. No push/PR/merge, default activation or parent/issue completion is claimed. + +### Managed checkout source within default application data (2026-11-07) + +Starting from b2f35a23, the actual App `/worktrees` default layout reproduced +`Private owner state cannot be a snapshot Location` during rewind preview. +Registration now permits only observation/private metadata reservation for its +ready source subtree; other data, the snapshot namespace and explicit private +roots remain protected. The native Store retains and revalidates registration. +Source-intent construction additionally requires the actual runtime-issued worker +and its verified complete native checkout set, matching activity, Session creation, +Location and registration. Registration alone cannot admit an unowned restore. +Git raw capture protects snapshot storage without silently excluding the registered +source subtree. No default Git source gate is enabled. + +The first application run after that correction stalled during managed native +admission. A live process sample showed no executing native worker; a bounded +confirmation regression then reproduced the timeout. Checkout policy lookup needs +the same Session mutex held by the admission caller. Ordinary restore and recovery +now pass an immutable captured Session revision across host admission without +retaining that mutex. Admission checks the current Store revision before reserving +native ownership; existing worker per-effect and consuming writer checks remain. +The stale-revision regression performs a real Session rename and proves refusal +publishes no native events and releases the admission flag. + +The candidate App test now creates a real managed linked checkout through the +public route in the default data directory, runs two scripted prompts, previews +and restores a tracked file, and checks the actual non-empty native checkout +resource publication against the consumed receipt digest. User index/config bytes +remain unchanged. Repeated explicit acknowledgement, including a fresh App over +the same Store, preserves later source symlinks/owner bytes, messages and inference +counts. This is App reopening rather than process death. The managed case and +three existing unmanaged root/scoped SHA-1/SHA-256 cases passed (two top-level +tests, 56.40 seconds). The first passing managed-only run took 15.54 seconds before +the final resource/reopen assertions were added; the final full App run includes +those assertions. + +Native registration tests prove unregistered under-data roots and explicit private +roots refuse, registered observations work, other data/link escapes remain private, +unowned journal preparation cannot publish an intent or mutate source, and changed +readiness invalidates the retained Location. All 47 worker tests passed in 4.69 +seconds. The default snapshot package passed 139 library, 18 file-copy and 22 Git +cases (library 60.63 seconds, Git 16.35 seconds); six library and one integration +subprocess fixtures were intentionally excluded from top-level selection. The +Windows-only and doc binaries select zero on this host. App/server/snapshot +all-target candidate Clippy passed with warnings denied. SDK generation consistency, +format/diff, strict OpenSpec (117 items) and spec lint (0 errors/25 warnings) passed. +Cargo test/Clippy commands used CARGO_INCREMENTAL=0 and --locked. Native sandbox +deprecations and the linker unwind-size warning were observed without main +attribution. The candidate snapshot package also passed 139 library, 18 file-copy +and 22 Git cases, with the same intentional fixture exclusions and zero selections; +its library/file-copy/Git times were 61.70/1.17/16.38 seconds. Both package runs +include the final native registration regression. + +Security review followed the code-review skill over the changed runtime/native +paths and their tests: the exception is registration-bound, source admission uses +retained native proof rather than IDs alone, and private boundaries and exact +metadata-only acknowledgements remain enforced. Full managed partial replacement, +keep/rollback/process cuts, partial construction/reclamation and native platform +acceptance remain open. No full workspace, CLI/TUI acceptance rerun, SDK execution, +source-bound CI, other-platform execution, push/PR/merge or parent/issue acceptance +is claimed by this increment. diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 79c5facc4..821b8cab5 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -958,3 +958,14 @@ fallback MAY conservatively widen only the process-local mutation-lock domain; it SHALL NOT create identity caches or grant Git validity, native source, permission, recovery or acknowledgement authority. Native source/Location and exact worker ownership checks remain independent and mandatory. + +Registered managed checkouts inside application data require a narrow source +exception: filesystem-only registration identification permits observation and +private metadata reservation, never source mutation. Retain the ready registration +and revalidate it with the native Location; protect the snapshot namespace, other +application data and explicit private roots. Before constructing a source intent, +the actual runtime-issued worker SHALL verify its complete retained native checkout +claims against enclosing registrations and exact activity/Session creation/Location. +Unowned callers SHALL refuse. Session policy lookup during native admission SHALL +run without the Session projection mutex held; immutable captured revisions and +existing per-effect/consuming writer checks SHALL fence concurrent changes. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index a47eafcc9..ead65d466 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -451,3 +451,15 @@ - **WHEN** root and scoped Locations identify a Git repository format unavailable to libgit2 discovery - **THEN** read-only ancestor marker discovery SHALL keep them in one conservative process-local mutation-lock domain without running Git or writing project identity caches - **AND** that lock domain SHALL NOT grant source, permission, recovery, acknowledgement or Git-validity authority + +#### Scenario: Registered managed checkout within application data +- **GIVEN** a ready registered linked checkout contains the exact snapshot Location within application data +- **WHEN** snapshot images or private worker metadata are prepared +- **THEN** that checkout's source subtree MAY be observed while the snapshot namespace, other application data and explicit private roots remain protected +- **AND** registration alone SHALL NOT authorize source mutation; the runtime-issued worker SHALL retain and verify the complete native checkout set, exact activity/Session creation/Location and current Session revision +- **AND** missing or changed registration, missing native claims and unowned restores SHALL refuse without source effects + +#### Scenario: Managed checkout policy lookup during worker admission +- **WHEN** native checkout admission reads Session policy during an owned restore or recovery +- **THEN** the caller SHALL release the Session projection mutex before that host call and retain an immutable captured revision +- **AND** a changed revision SHALL refuse worker admission or subsequent source effects, preserving existing native evidence diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index a20138651..0871c70cc 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -600,3 +600,16 @@ this is coordination, never source/permission authority. Add candidate jobs to t native matrix, keeping their evidence distinct. Default admission, complete managed resource/storage-root integration, keep/rollback/process-death/runtime cuts, compatibility/reclamation and full platform acceptance remain open with M2–M8. + +Managed default-storage continuation: identify ready registered source subtrees +within application data for observation/reservation while retaining private +snapshot/other-data/explicit-root boundaries. Require the actual worker's complete +native checkout claims before constructing source intents; registration alone +cannot authorize an unowned restore. Release the Session projection mutex before +host checkout-policy admission, retain its captured revision and refuse stale +admission. The non-default App candidate now exercises actual default /worktrees +creation, native resource publication, restore, unchanged user Git metadata and +reopened metadata acknowledgement without source replay. This is an ordinary +managed restore positive, not complete managed keep/rollback/process-death, +construction/reclamation/platform acceptance. Default Git admission remains off; +M2–M8 and all parent tasks remain open. From b75315e0eadd878571d66dace278ed8d150b4756 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 15:28:30 -0400 Subject: [PATCH 50/87] Keep snapshot warnings nonfatal in JSONL exec --- crates/cyber-cli/src/commands/exec/driver.rs | 8 ++++++++ crates/cyber-cli/tests/exec_stream_input.rs | 19 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/cyber-cli/src/commands/exec/driver.rs b/crates/cyber-cli/src/commands/exec/driver.rs index 2442c132d..e53b3eb58 100644 --- a/crates/cyber-cli/src/commands/exec/driver.rs +++ b/crates/cyber-cli/src/commands/exec/driver.rs @@ -412,6 +412,14 @@ impl Driver<'_> { json!({ "message_id": event.data["message_id"], "text": event.data["text"] }), ); } + "session.error" if event.data["kind"] == "snapshot" => { + eprintln!( + "snapshot warning: {}", + event.data["message"] + .as_str() + .unwrap_or("code rewind unavailable") + ); + } "session.error" => self.failed(&event.data), "session.retry" => self.out.retry(&event.data), _ => self.out.durable(&mut self.run, kind, &event.data), diff --git a/crates/cyber-cli/tests/exec_stream_input.rs b/crates/cyber-cli/tests/exec_stream_input.rs index 68298084a..8618609c3 100644 --- a/crates/cyber-cli/tests/exec_stream_input.rs +++ b/crates/cyber-cli/tests/exec_stream_input.rs @@ -230,6 +230,23 @@ impl Driven { } } +#[test] +fn unavailable_snapshots_do_not_fail_a_successful_jsonl_input() { + let provider = Provider::start(&["text:completed without snapshots"]); + // Driven creates an invalid Git marker: capture warns, but inference succeeds. + let mut run = Driven::start(&provider, &[]); + let mut seen = Vec::new(); + run.send(r#"{"type":"user","text":"one","id":"warning"}"#); + let result = run.until("result", &mut seen); + assert_eq!(result["id"], "warning"); + assert_eq!(result["result"], "completed without snapshots"); + assert_eq!(result["error"], Value::Null, "{seen:#?}"); + assert_eq!(result["exit_code"], 0, "{seen:#?}"); + run.send(r#"{"type":"end"}"#); + assert_eq!(run.until("end", &mut seen)["exit_code"], 0); + assert_eq!(run.exit(), Some(0)); +} + #[test] fn two_prompts_give_two_results_in_order_then_end() { let provider = Provider::start(&["text:first answer", "text:second answer"]); @@ -245,7 +262,7 @@ fn two_prompts_give_two_results_in_order_then_end() { (first["id"].as_str(), first["result"].as_str()), (Some("a"), Some("first answer")) ); - assert_eq!(first["exit_code"], 0); + assert_eq!(first["exit_code"], 0, "{seen:#?}"); run.send(r#"{"type":"user","text":"two","id":"b"}"#); run.send(r#"{"type":"end"}"#); let second = run.until("result", &mut seen); From 3d3471f96a63fd6e79e043d454b46fa7f2058b2f Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 15:35:08 -0400 Subject: [PATCH 51/87] Reclaim managed Git payloads with retained checkout proofs --- .github/workflows/non-git-snapshot-proof.yml | 19 +- .../cyber-app/tests/snapshot_git_runtime.rs | 87 ++++++ crates/cyber-cli/Cargo.toml | 2 + crates/cyber-cli/tests/branches_cli.rs | 295 +++++++++++++++++- .../activity/snapshot/acknowledgement.rs | 17 + .../snapshot/acknowledgement/tests.rs | 29 ++ .../src/runtime/snapshot_reclamation.rs | 17 +- crates/cyber-snapshot/src/copy/git/tests.rs | 63 ++++ crates/cyber-snapshot/src/copy/lifetime.rs | 15 +- crates/cyber-snapshot/src/copy/reclamation.rs | 38 +-- .../src/copy/reclamation/cancelled.rs | 15 +- .../src/copy/reclamation/checkouts.rs | 71 +++++ .../src/copy/reclamation/paired.rs | 39 +-- crates/cyber-snapshot/src/git/prepared.rs | 37 ++- docs/goal-delivery.md | 69 ++++ .../managed-recovery.md | 25 ++ .../specs/snapshots-checkpoints/spec.md | 20 ++ .../complete-daily-coding-surfaces/tasks.md | 80 ++++- 18 files changed, 850 insertions(+), 88 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/reclamation/checkouts.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 8bdaa9140..4a3141ca7 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -10,9 +10,14 @@ on: - crates/cyber-server/src/runtime/rewind*.rs - crates/cyber-server/src/http/sessions.rs - crates/cyber-app/tests/snapshot_git_runtime.rs + - crates/cyber-cli/Cargo.toml + - crates/cyber-cli/src/** + - crates/cyber-cli/tests/branches_cli.rs + - crates/cyber-cli/tests/support/** - crates/cyber-server/src/workspace.rs - crates/cyber-core/src/binary/native/** - crates/cyber-core/src/binary/native.rs + - crates/cyber-core/src/worktrees/** - Cargo.lock - .github/workflows/non-git-snapshot-proof.yml workflow_dispatch: @@ -23,7 +28,7 @@ concurrency: cancel-in-progress: true jobs: native: - name: File-copy snapshots (${{ matrix.os }}) + name: Snapshot native (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 45 strategy: @@ -45,12 +50,22 @@ jobs: run: | set -euo pipefail cargo test --locked -p cyber-snapshot -- --nocapture 2>&1 | tee non-git-snapshot-proof.log + - name: Read-only checkout acknowledgement requirement + shell: bash + run: | + set -euo pipefail + cargo test --locked -p cyber-core --lib worktrees::activity::snapshot::acknowledgement::tests::maintenance_requires_an_existing_acknowledgement_without_publishing_one -- --exact --nocapture 2>&1 | tee managed-checkout-ack-proof.log - name: Owned Git runtime acceptance candidate shell: bash run: | set -euo pipefail cargo test --locked -p cyber-snapshot --features native-git-acceptance -- --nocapture 2>&1 | tee git-adapter-proof.log cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_git_runtime -- --nocapture 2>&1 | tee git-runtime-proof.log + - name: Managed Git CLI process-death recovery candidate + shell: bash + run: | + set -euo pipefail + cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_git_keep_and_rollback_after_real_partial_restore_kill -- --nocapture 2>&1 | tee git-cli-recovery-proof.log - name: Retain native evidence if: always() uses: actions/upload-artifact@v4 @@ -58,6 +73,8 @@ jobs: name: non-git-snapshot-proof-${{ matrix.os }} path: | non-git-snapshot-proof.log + managed-checkout-ack-proof.log git-adapter-proof.log git-runtime-proof.log + git-cli-recovery-proof.log if-no-files-found: warn diff --git a/crates/cyber-app/tests/snapshot_git_runtime.rs b/crates/cyber-app/tests/snapshot_git_runtime.rs index 7030bf9fc..cda5f9029 100644 --- a/crates/cyber-app/tests/snapshot_git_runtime.rs +++ b/crates/cyber-app/tests/snapshot_git_runtime.rs @@ -283,9 +283,96 @@ async fn managed_worktree_in_default_data_directory_restores_through_owned_worke config ); reconcile_after_owner_edit(&f, &receipt).await; + reclaim_managed_restore(&f, &receipt).await; + assert_eq!(std::fs::read(&index_path).unwrap(), index); + assert_eq!( + std::fs::read(f.project.join(".git/config")).unwrap(), + config + ); f.app.runtime.shutdown().await; } +async fn reclaim_managed_restore(f: &Fixture, receipt: &SnapshotRestoreReceipt) { + use cyber_server::runtime::snapshot_reclamation::Receipt; + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + let snapshots = cyber_snapshot::GitSnapshots::with_reclamation( + f.paths.data.clone(), + Arc::new(|_| json!({"snapshots":{"retention_days":7}})), + vec![ + f.paths.config.clone(), + f.paths.state.clone(), + f.paths.cache.clone(), + ], + None, + Some(policy.clone()), + ); + let journal = f.journal(receipt); + let intent = std::fs::read(journal.join("intent.json")).unwrap(); + let inference = f.model.requests().len(); + assert_eq!( + snapshots + .reclaim(f.directory.to_str().unwrap()) + .await + .unwrap(), + 0 + ); + let owner = receipt.origin.activity_id.clone(); + f.app + .store + .transaction(move |db| { + db.execute( + "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", + rusqlite::params![8i64 * 86400000, owner], + )?; + Ok(()) + }) + .unwrap(); + assert!( + snapshots + .reclaim(f.directory.to_str().unwrap()) + .await + .unwrap() + > 0 + ); + let record = policy + .record(&Receipt::Restore(receipt.clone())) + .unwrap() + .unwrap(); + assert!(record.completed); + assert_eq!( + snapshots + .reclaim(f.directory.to_str().unwrap()) + .await + .unwrap(), + 0 + ); + assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); + assert_eq!(std::fs::read_dir(journal).unwrap().count(), 1); + assert_reclaimed_source_and_receipt(f, receipt, inference).await; +} + +async fn assert_reclaimed_source_and_receipt( + f: &Fixture, + receipt: &SnapshotRestoreReceipt, + inference: usize, +) { + let reopened = build(&f.paths, &f.directory, f._temp.path()).await; + let client = Client::embedded(reopened.embedded()).at(f.directory.to_str().unwrap()); + receipt_roundtrip(&client, &f.session, receipt).await; + assert!( + std::fs::symlink_metadata(f.directory.join("tracked")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_to_string(f._temp.path().join("later-owner")).unwrap(), + "later owner bytes" + ); + assert_eq!(f.model.requests().len(), inference); + reopened.runtime.shutdown().await; +} + #[tokio::test] async fn actual_git_worker_commits_receipt_and_reconciles_without_source_replay() { for (format, scoped) in [("sha1", false), ("sha1", true), ("sha256", true)] { diff --git a/crates/cyber-cli/Cargo.toml b/crates/cyber-cli/Cargo.toml index 05b4c95f3..935503dbc 100644 --- a/crates/cyber-cli/Cargo.toml +++ b/crates/cyber-cli/Cargo.toml @@ -15,6 +15,8 @@ path = "src/main.rs" default = ["otlp-grpc"] # OTLP export over gRPC (`telemetry.otel.protocol: grpc`). otlp-grpc = ["cyber-telemetry/otlp-grpc"] +# Explicit native Git acceptance only; release/default admission stays gated. +native-git-acceptance = ["cyber-app/native-git-acceptance"] [dependencies] cyber-collab = { path = "../cyber-collab" } diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index b8ef787be..4f3a87c41 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -45,6 +45,7 @@ struct Env { _dir: tempfile::TempDir, root: PathBuf, catalog: PathBuf, + directory: PathBuf, } impl Env { @@ -69,6 +70,7 @@ impl Env { _dir: dir, root, catalog, + directory: repo, } } @@ -81,7 +83,7 @@ impl Env { .env("CYBER_HOME", self.root.join("cyber-home")) .env("CYBER_MODELS_PATH", &self.catalog) .env("PATH", std::env::var("PATH").unwrap_or_default()) - .current_dir(self.root.join("repo")); + .current_dir(&self.directory); #[cfg(windows)] if let Some(system) = std::env::var_os("SystemRoot") { command.env("SystemRoot", system); @@ -94,7 +96,7 @@ impl Env { } fn file(&self, name: &str) -> Option { - std::fs::read_to_string(self.root.join("repo").join(name)).ok() + std::fs::read_to_string(self.directory.join(name)).ok() } /// Run one prompt in `session` (a new Session when `None`); returns the Session ID. @@ -496,7 +498,7 @@ fn conversation_rewind_works_without_git_and_stale_context_never_applies() { /// Use the same authenticated reviewed actions as the TUI, through the actual CLI. fn reviewed_selection(env: &Env, message: &str, target: &str, action: &str) -> Value { json!({"message_id":message,"target":target,"action":action, - "expected_directory":env.root.join("repo").display().to_string()}) + "expected_directory":env.directory.display().to_string()}) } fn reviewed_preview(env: &Env, id: &str, selection: &Value) -> Value { json(&env.cyber(&[ @@ -541,13 +543,18 @@ fn kill_registered_service(env: &Env) { .output() .unwrap(); assert!(killed.status.success(), "{killed:?}"); - let address = registration["url"] - .as_str() - .unwrap() - .trim_start_matches("http://") - .trim_end_matches('/'); + let registration: cyber_app::Registration = serde_json::from_value(registration).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); let deadline = std::time::Instant::now() + Duration::from_secs(10); - while std::net::TcpStream::connect(address).is_ok() { + // Parallel fixtures may reuse an ephemeral TCP port after this process + // exits. Probe the recorded server ID, never the presence of any listener. + while runtime + .block_on(cyber_app::probe_registration(®istration)) + .is_some_and(|probe| probe.id.as_deref() == Some(registration.id.as_str())) + { assert!( std::time::Instant::now() < deadline, "killed fixture service still answers" @@ -791,6 +798,242 @@ fn managed_keep_and_rollback_after_real_partial_restore_kill_preserve_owner_file managed_native_recovery_after_partial_kill("rollback"); } +#[test] +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn managed_git_keep_and_rollback_after_real_partial_restore_kill() { + for action in ["keep", "rollback"] { + managed_git_partial_kill(action); + } +} + +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn git_fixture(directory: &std::path::Path, args: &[&str]) -> String { + let output = Command::new("git") + .arg("-C") + .arg(directory) + .args(args) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + String::from_utf8(output.stdout).unwrap().trim().to_owned() +} + +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn managed_git_partial_kill(action: &str) { + let model = scripted_model::ScriptedModel::start(reply); + let mut env = Env::new(&model); + git_fixture( + &env.directory, + &[ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "--quiet", + "-m", + "Initial", + ], + ); + let created = json(&env.cyber(&[ + "api", + "POST", + "/worktrees", + "-d", + r#"{"name":"managed","session":{"model":"test/main","mode":"accept-edits"}}"#, + ])); + let id = created["data"]["session"]["id"] + .as_str() + .unwrap() + .to_owned(); + env.directory = PathBuf::from(created["data"]["worktree"]["path"].as_str().unwrap()); + assert!( + env.directory + .starts_with(env.root.join("cyber-home/data/worktrees")) + ); + for index in 0..1280 { + std::fs::write(env.directory.join(format!("bulk_{index:04}")), "before").unwrap(); + } + std::fs::write(env.directory.join("zz\nfixture"), "before").unwrap(); + git_fixture(&env.directory, &["add", "--all"]); + let index_path = PathBuf::from(git_fixture( + &env.directory, + &["rev-parse", "--path-format=absolute", "--git-path", "index"], + )); + let index = std::fs::read(&index_path).unwrap(); + let config_path = env.root.join("repo/.git/config"); + let config = std::fs::read(&config_path).unwrap(); + env.exec(Some(&id), "before snapshot"); + let first = env.user_ids(&id)[0].clone(); + for index in 0..1280 { + std::fs::write(env.directory.join(format!("bulk_{index:04}")), "after").unwrap(); + } + std::fs::write(env.directory.join("zz\nfixture"), "after").unwrap(); + std::fs::write(env.directory.join("aa_removed"), "after-only source").unwrap(); + env.exec(Some(&id), "after snapshot"); + let messages = kill_partial_restore(&env, &id, &first); + let requests = model.requests().len(); + std::fs::write(env.directory.join("unrelated-owner"), "owner data").unwrap(); + if action == "keep" { + std::fs::write(env.directory.join("bulk_1279"), "later owner image").unwrap(); + } + let partial: Vec<_> = (0..1280) + .map(|index| env.file(&format!("bulk_{index:04}")).unwrap()) + .collect(); + let (journal, review) = reviewed_recovery_cli(&env, &id, action); + assert_eq!( + review["origin"]["worktree_id"], + created["data"]["worktree"]["id"] + ); + let request_id = format!("managed-git-kill-{action}"); + let request = json!({"journal_id":journal,"action":action,"confirmation":review["confirmation"],"request_id":request_id}); + let mut stale = request.clone(); + stale["confirmation"] = "0".repeat(64).into(); + assert!(!recovery_confirm_cli(&env, &id, &stale).status.success()); + let result = json(&recovery_confirm_cli(&env, &id, &request)); + assert_eq!(result["state"], "acknowledged"); + assert_eq!(result["journal_acknowledged"], true); + assert_managed_git_images(&env, action, &partial); + assert_foreign_recovery_request_refused(&env, &request); + assert_eq!(env.messages(&id), messages); + assert_native_settlement_link(&env, &journal, &result); + assert_recovery_lookup_does_not_replay(&env, &id, &request_id, &request, &result, &messages); + assert_managed_git_reclamation(&env, &result); + assert_eq!(std::fs::read(index_path).unwrap(), index); + assert_eq!(std::fs::read(config_path).unwrap(), config); + assert_eq!(model.requests().len(), requests); +} + +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn assert_managed_git_reclamation(env: &Env, result: &Value) { + use cyber_server::runtime::{SnapshotRecoveryReceipt, snapshot_reclamation::Receipt}; + exits(&env.cyber(&["service", "stop"]), 0); + let maintenance = reclamation_support::Maintenance::open(&env.root.join("cyber-home")); + let receipt: SnapshotRecoveryReceipt = + serde_json::from_value(result["receipt"].clone()).unwrap(); + let checkout = receipt.checkout.as_ref().unwrap(); + let original = checkout.original.as_ref().unwrap(); + let phase: Value = serde_json::from_slice( + &std::fs::read( + maintenance + .namespace + .join("workers") + .join(original.lifetime.source_phase.as_ref().unwrap()), + ) + .unwrap(), + ) + .unwrap(); + let bindings: Vec = + serde_json::from_value(phase["checkout_bindings"].clone()).unwrap(); + assert!(!bindings.is_empty()); + let binding = &bindings[0]; + let marker = binding + .common_dir + .join("cyber-worktree-activity") + .join(&binding.worktree_id) + .join(format!("{}.snapshot-ack", binding.owner_id)); + let hidden = env.root.join("retained-checkout-marker"); + let source = env.file("bulk_0000"); + assert_eq!(maintenance.reclaim(&env.directory).unwrap(), 0); + maintenance.age(checkout.resolution_origin.activity_id.clone()); + let resources = original.lifetime.checkout_owner().unwrap(); + let receipt = Receipt::Recovery(receipt.clone()); + std::fs::rename(&marker, &hidden).unwrap(); + let refused = maintenance.reclaim(&env.directory).unwrap_err(); + assert!( + refused.contains("acknowledgement has not been published"), + "{refused}" + ); + assert!( + !marker.exists(), + "cleanup must not publish a missing marker" + ); + assert!(maintenance.policy.record(&receipt).unwrap().is_none()); + std::fs::rename(&hidden, &marker).unwrap(); + let locations = + cyber_core::worktrees::Repository::managed_locations_at(&env.directory).unwrap(); + let live = locations[0] + .0 + .reacquire_snapshot(&locations[0].1, binding, &resources) + .unwrap(); + let refused = maintenance.reclaim(&env.directory).unwrap_err(); + assert!(refused.contains("worker is still active"), "{refused}"); + assert!(maintenance.policy.record(&receipt).unwrap().is_none()); + drop(live); + let cut = maintenance.plan_only(&env.directory).unwrap_err(); + assert!( + cut.contains("fixture cut after durable paired plan"), + "{cut}" + ); + assert!( + !maintenance + .policy + .record(&receipt) + .unwrap() + .unwrap() + .completed + ); + assert!(maintenance.reclaim(&env.directory).unwrap() > 0); + assert!( + maintenance + .policy + .record(&receipt) + .unwrap() + .unwrap() + .completed + ); + assert_eq!(maintenance.reclaim(&env.directory).unwrap(), 0); + assert_eq!(env.file("bulk_0000"), source); + assert!(marker.exists()); +} + +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn assert_managed_git_images(env: &Env, action: &str, partial: &[String]) { + assert_eq!( + env.file("bulk_0000").as_deref(), + Some(if action == "rollback" { + "after" + } else { + "before" + }) + ); + assert_eq!(env.file("unrelated-owner").as_deref(), Some("owner data")); + assert_eq!(env.file("zz\nfixture").as_deref(), Some("after")); + for (index, before) in partial.iter().enumerate() { + assert_eq!( + env.file(&format!("bulk_{index:04}")).as_deref(), + Some(if action == "rollback" { + "after" + } else { + before + }), + "{action}: bulk_{index:04}" + ); + } + assert_eq!( + env.file("aa_removed").as_deref(), + (action == "rollback").then_some("after-only source") + ); +} + #[test] #[cfg(any(target_os = "macos", target_os = "linux"))] fn exited_pre_origin_preparation_is_reviewed_cancelled_and_lookup_only_after_restart() { @@ -1474,6 +1717,10 @@ fn kill_after_prepared_images(env: &Env, id: &str, first: &str) -> String { .join("journals") .join(journal_id) .join("intent.json"); + // Origin publication and staging 80 MiB are separate bounded phases. A + // loaded runner may spend most of the origin-observation deadline before + // staging starts. The SQLite writer still fences every source admission. + let deadline = std::time::Instant::now() + Duration::from_secs(120); loop { let prepared = std::fs::read(&intent) .ok() @@ -1485,9 +1732,16 @@ fn kill_after_prepared_images(env: &Env, id: &str, first: &str) -> String { if prepared { break; } + if let Some(status) = pending.try_wait().unwrap() { + let output = pending.wait_with_output().unwrap(); + panic!( + "restore exited before publishing prepared identities ({status}): {}", + String::from_utf8_lossy(&output.stderr) + ); + } assert!( - pending.try_wait().unwrap().is_none() && std::time::Instant::now() < deadline, - "prepared identities were not published" + std::time::Instant::now() < deadline, + "prepared identities were not published within the staging deadline" ); std::thread::sleep(Duration::from_millis(2)); } @@ -2034,10 +2288,21 @@ fn assert_recovery_checkout_lifetimes(namespace: &std::path::Path, receipt: &Val ) .unwrap(); assert_eq!(phase["lifetime"], evidence["lifetime"]); - assert_eq!(phase["checkout_bindings"], serde_json::json!([])); + let bindings: Vec = + serde_json::from_value(phase["checkout_bindings"].clone()).unwrap(); + if let Some(worktree) = &lifetime.owner.worktree_id { + assert!(!bindings.is_empty()); + assert!( + bindings + .iter() + .any(|binding| &binding.worktree_id == worktree) + ); + } else { + assert!(bindings.is_empty()); + } assert_eq!( evidence["resources_digest"], - lifetime.checkout_resources_digest(&[]).unwrap() + lifetime.checkout_resources_digest(&bindings).unwrap() ); } } @@ -2051,7 +2316,7 @@ fn assert_recovery_lookup_does_not_replay( result: &Value, messages: &[Value], ) { - let repo = env.root.join("repo"); + let repo = &env.directory; std::fs::write(repo.join("bulk_0000"), "new owner edit after recovery").unwrap(); let retry = json(&recovery_confirm_cli(env, id, request)); assert_eq!(retry["state"], "acknowledged"); @@ -2128,7 +2393,7 @@ fn reconcile_missing_journal_acknowledgement( } let target = env.root.join("ack-owner-target"); std::fs::write(&target, "new owner edit after recovery").unwrap(); - let source = env.root.join("repo/bulk_0000"); + let source = env.directory.join("bulk_0000"); std::fs::remove_file(&source).unwrap(); std::os::unix::fs::symlink(&target, &source).unwrap(); let reconciled = json(&env.cyber(&[ diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs index dd2bafcb2..e4155fa91 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs @@ -200,6 +200,23 @@ impl CheckoutAcknowledgement { Err(error) => Err(error), } } + /// Retain a previously published exact disposition without creating one. + /// Maintenance requires this stronger check: missing metadata is not an + /// acknowledgement and must never be repaired as a side effect of cleanup. + pub fn require_existing(&self, claim: &mut SnapshotClaim) -> io::Result<()> { + self.retain_existing(claim)?; + if claim.acknowledgement.is_none() { + return Err(invalid("Checkout acknowledgement has not been published")); + } + // An exited publisher may have died after rename and before its final + // directory sync. Flush the exact retained marker before maintenance + // consumes it; this never creates or rewrites acknowledgement metadata. + let acknowledged = claim.acknowledgement.as_ref().expect("existing marker"); + let pins = claim.lease.snapshot_pins.as_ref().expect("retained pins"); + super::publication::sync_file(&acknowledged.file)?; + super::publication::sync_directory(&pins.directory)?; + claim.verify() + } fn matches(&self, record: &Record) -> io::Result<()> { if record.binding != self.binding || record.resources_digest != self.resources_digest diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs index 89a730b2e..d35c6b7db 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement/tests.rs @@ -50,6 +50,35 @@ fn authority(binding: &CheckoutBinding) -> CheckoutAcknowledgement { CheckoutAcknowledgement::new(binding, &"a".repeat(64), &"b".repeat(64)).unwrap() } +#[test] +fn maintenance_requires_an_existing_acknowledgement_without_publishing_one() { + let temp = tempfile::tempdir().unwrap(); + let managed = fixture(&temp.path().canonicalize().unwrap()); + let binding = original(&managed); + let mut proof = claim(&managed, &binding); + let authority = authority(&binding); + let marker = directory(&managed).join("op_fixture.snapshot-ack"); + let original_path = directory(&managed).join("op_fixture.lock"); + let bytes = std::fs::read(&original_path).unwrap(); + assert!(authority.require_existing(&mut proof).is_err()); + assert!(!marker.exists()); + assert_eq!(std::fs::read(&original_path).unwrap(), bytes); + authority.publish(&mut proof).unwrap(); + drop(proof); + let mut reloaded = claim(&managed, &binding); + authority.require_existing(&mut reloaded).unwrap(); + let foreign = CheckoutAcknowledgement::new(&binding, &"a".repeat(64), &"c".repeat(64)).unwrap(); + assert!(foreign.require_existing(&mut reloaded).is_err()); + let hidden = temp.path().join("retained-marker"); + std::fs::rename(&marker, &hidden).unwrap(); + assert!(reloaded.verify().is_err()); + assert!( + !marker.exists(), + "verification must not recreate the marker" + ); + assert_eq!(std::fs::read(&original_path).unwrap(), bytes); +} + #[test] fn acknowledgement_is_exact_idempotent_and_preserves_original_and_independent_unknown_bytes() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation.rs b/crates/cyber-server/src/runtime/snapshot_reclamation.rs index 85acbfbdb..5aea49e21 100644 --- a/crates/cyber-server/src/runtime/snapshot_reclamation.rs +++ b/crates/cyber-server/src/runtime/snapshot_reclamation.rs @@ -166,19 +166,10 @@ impl Plan { } pub fn validate(&self) -> Result<(), StoreError> { let owners = self.receipt.owners()?; - for owner in &owners { - if owner.origin.worktree_id.is_some() - || owner.resources_digest - != owner - .lifetime - .checkout_resources_digest(&[]) - .map_err(invalid)? - { - return Err(invalid( - "Managed reclamation requires retained resource guards", - )); - } - } + // Receipt validation binds the resource digest. Its actual complete + // native checkout set and published dispositions are retained by the + // adapter's deletion closure, alongside namespace and lifetime pins. + // Shape validation alone never grants a filesystem capability. let version = match &self.receipt { Receipt::Restore(_) => 1, Receipt::Recovery(_) => 2, diff --git a/crates/cyber-snapshot/src/copy/git/tests.rs b/crates/cyber-snapshot/src/copy/git/tests.rs index b7464c905..e84613cb1 100644 --- a/crates/cyber-snapshot/src/copy/git/tests.rs +++ b/crates/cyber-snapshot/src/copy/git/tests.rs @@ -11,6 +11,69 @@ struct Repo { snapshots: Arc, } +#[tokio::test] +async fn repeated_git_blob_images_charge_every_placement_and_do_not_survive_a_new_handoff() { + for format in ["sha1", "sha256"] { + let repo = Repo::with_format(format); + for index in 0..16 { + std::fs::write(repo.root.join(format!("entry_{index:02}")), b"same").unwrap(); + } + std::fs::set_permissions( + repo.root.join("entry_00"), + std::fs::Permissions::from_mode(0o755), + ) + .unwrap(); + let tree = repo.track(&repo.root).await; + let shadow = repo.snapshots.shadow(repo.root.to_str().unwrap()).unwrap(); + let request = SnapshotRestoreRequest { + directory: repo.root.to_str().unwrap().into(), + target: tree.clone(), + recorded: tree.clone(), + expected: tree, + paths: None, + }; + let prepared = shadow + .prepare_restore(&request, 4, 16 * 4 * 3) + .await + .unwrap(); + for tree in [&prepared.target, &prepared.recorded, &prepared.expected] { + assert_eq!(tree.images.len(), 16); + assert_eq!(tree.images["entry_00"].mode, "100755"); + assert_eq!(tree.images["entry_01"].mode, "100644"); + for image in tree.images.values() { + assert_eq!(image.bytes, b"same"); + image.verify().unwrap(); + } + } + assert!( + shadow + .prepare_restore(&request, 4, 16 * 4 * 3 - 1) + .await + .is_err() + ); + assert!( + shadow + .prepare_restore(&request, 3, 16 * 4 * 3) + .await + .is_err() + ); + let object = &prepared.target.images["entry_00"].object; + let path = shadow + .git_dir + .join("objects") + .join(&object[..2]) + .join(&object[2..]); + std::fs::remove_file(&path).unwrap(); + std::fs::write(path, b"corrupt after earlier handoff").unwrap(); + assert!( + shadow + .prepare_restore(&request, 4, 16 * 4 * 3) + .await + .is_err() + ); + } +} + #[test] fn unsupported_native_config_environment_keeps_ordinary_git_capture_available() { let output = std::process::Command::new(std::env::current_exe().unwrap()) diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index 2d263d145..a61e45df3 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -274,7 +274,6 @@ impl Store { .ok_or("Missing acquired reclamation resources")?; if phase.admitted != admitted || phase.origin.as_ref() != admitted.then_some(origin) - || !resources.is_empty() || binding.checkout_resources_digest(resources)? != resources_digest { return Err("Reclamation source phase contradicts committed restore evidence".into()); @@ -669,6 +668,20 @@ mod tests { Some(super::super::ownership::Ownership::prepare(&store, &directory, &[]).unwrap()); Store::record(&directory, &journal).unwrap(); lifetime.admit_source(&origin).unwrap(); + let digest = binding.checkout_resources_digest(&resources).unwrap(); + store + .verify_reclamation_phase(&binding, &origin, &digest) + .unwrap(); + assert!( + store + .verify_reclamation_phase(&binding, &origin, &"f".repeat(64)) + .is_err() + ); + assert!( + store + .verify_reclamation_phase_admission(&binding, &origin, &digest, false) + .is_err() + ); assert!( store .record_checkout_bindings(&binding, &resources) diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index 4d625fe81..876b691b0 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -1,5 +1,6 @@ //! Reclaim only exact acknowledged displaced images; keep intent/lifetime receipts. mod cancelled; +mod checkouts; mod paired; use super::{ Store, ownership, @@ -20,6 +21,7 @@ use std::{ }; struct Prepared { + checkouts: checkouts::Checkouts, store: Store, journal: Dir, intent: MetadataPin, @@ -82,11 +84,7 @@ fn ordinary(record: &Journal) -> bool { && record.origin.is_some() && record.resolution.is_none() && record.settled_by.is_none() - && record - .context - .as_ref() - .is_some_and(|context| context.git.is_none()) - && record.backend != Some(JournalBackend::Git) + && record.context.is_some() && record .changes .iter() @@ -213,15 +211,13 @@ fn prepare( .as_ref() .ok_or("Legacy restore has no cleanup checkout proof")?; let lifetime = store.lock_lifetime(&checkout.lifetime)?; - let resources = lifetime - .checkout_bindings()? - .ok_or("Missing acquired checkout inventory")?; - if !resources.is_empty() - || checkout.resources_digest != checkout.lifetime.checkout_resources_digest(&resources)? - { - return Err("Managed checkout reclamation requires retained resource guards".into()); - } - unmanaged(&store)?; + let checkouts = checkouts::Checkouts::acquire( + &store, + &lifetime, + &checkout.lifetime, + &checkout.resources_digest, + &checkouts::receipt_digest(&receipt)?, + )?; let ownership = record .ownership .as_ref() @@ -315,6 +311,7 @@ fn prepare( true, )?; Ok(Prepared { + checkouts, store, journal, intent, @@ -329,15 +326,6 @@ fn prepare( intent_digest, }) } -fn unmanaged(store: &Store) -> Result<(), String> { - if !cyber_core::worktrees::Repository::managed_locations_at(&store.root) - .map_err(storage::fail)? - .is_empty() - { - return Err("Snapshot cleanup checkout inventory changed".into()); - } - Ok(()) -} impl Prepared { fn verify_routes(&self) -> Result<(), String> { let _reader_lease = &self.workspace; @@ -359,7 +347,7 @@ impl Prepared { .as_deref() .ok_or("Missing source phase route")?, )?; - unmanaged(&self.store) + self.checkouts.verify(&self.store) } fn verify(&self) -> Result<(), String> { self.verify_routes()?; @@ -401,7 +389,7 @@ impl Prepared { { return Err("Unknown journal entry appeared before reclamation".into()); } - unmanaged(&self.store) + self.checkouts.verify(&self.store) } fn delete(self) -> Result { self.verify()?; diff --git a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs index 5736e1fbb..5a8ab7146 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs @@ -93,17 +93,17 @@ fn prepare( .ok_or("Missing preparation journal origin")?; store.verify_worker_namespace(origin)?; let lifetime = store.lock_lifetime(&receipt.lifetime)?; - let resources = lifetime - .checkout_bindings()? - .ok_or("Missing preparation checkout inventory")?; let digest = receipt .checkout_resources_digest .as_deref() .ok_or("Missing consumed preparation resources")?; - if !resources.is_empty() || receipt.lifetime.checkout_resources_digest(&resources)? != digest { - return Err("Managed preparation cleanup requires resource guards".into()); - } - unmanaged(&store)?; + let checkouts = checkouts::Checkouts::acquire( + &store, + &lifetime, + &receipt.lifetime, + digest, + &checkouts::receipt_digest(&settlement.receipt)?, + )?; let workers = storage::existing_child(&store.namespace, "workers", true)?; let phase_name = receipt .lifetime @@ -133,6 +133,7 @@ fn prepare( return Err("Preparation cleanup differs from retained plan".into()); } let prepared = Prepared { + checkouts, store, journal, intent, diff --git a/crates/cyber-snapshot/src/copy/reclamation/checkouts.rs b/crates/cyber-snapshot/src/copy/reclamation/checkouts.rs new file mode 100644 index 000000000..c8670fea0 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/reclamation/checkouts.rs @@ -0,0 +1,71 @@ +//! Read-only reacquisition of every consumed checkout and its published disposition. +use super::*; +use cyber_core::worktrees::{CheckoutAcknowledgement, Managed, Repository, SnapshotClaim}; +use cyber_server::runtime::SnapshotLifetimeBinding; +use cyber_server::runtime::snapshot_reclamation::Receipt; + +pub(super) struct Checkouts { + locations: Vec, + claims: Vec, +} + +impl Checkouts { + pub fn acquire( + store: &Store, + lifetime: &super::super::lifetime::Lifetime, + binding: &SnapshotLifetimeBinding, + resources_digest: &str, + receipt_digest: &str, + ) -> Result { + let resources = lifetime + .checkout_bindings()? + .ok_or("Missing reclamation checkout inventory")?; + if binding.checkout_resources_digest(&resources)? != resources_digest { + return Err("Reclamation checkout resource digest changed".into()); + } + let locations = Repository::managed_locations_at(&store.root).map_err(storage::fail)?; + if locations.len() != resources.len() { + return Err("Reclamation checkout inventory changed".into()); + } + let owner = binding.checkout_owner()?; + let mut claims = Vec::with_capacity(resources.len()); + for ((repository, managed), resource) in locations.iter().zip(&resources) { + let mut claim = repository + .reacquire_snapshot(managed, resource, &owner) + .map_err(storage::fail)?; + CheckoutAcknowledgement::new(resource, resources_digest, receipt_digest) + .and_then(|ack| ack.require_existing(&mut claim)) + .map_err(storage::fail)?; + claims.push(claim); + } + let guards = Self { + locations: locations.into_iter().map(|(_, managed)| managed).collect(), + claims, + }; + guards.verify(store)?; + Ok(guards) + } + + pub fn verify(&self, store: &Store) -> Result<(), String> { + let current = Repository::managed_locations_at(&store.root).map_err(storage::fail)?; + if !current + .iter() + .map(|(_, managed)| managed) + .eq(&self.locations) + { + return Err("Reclamation checkout registration changed".into()); + } + for claim in &self.claims { + claim.verify().map_err(storage::fail)?; + } + Ok(()) + } +} + +pub(super) fn receipt_digest(receipt: &Receipt) -> Result { + match receipt { + Receipt::Restore(receipt) => receipt.checkout_acknowledgement_digest(), + Receipt::Recovery(receipt) => receipt.checkout_acknowledgement_digest(), + Receipt::Preparation(receipt) => receipt.checkout_acknowledgement_digest(), + } +} diff --git a/crates/cyber-snapshot/src/copy/reclamation/paired.rs b/crates/cyber-snapshot/src/copy/reclamation/paired.rs index 6f30f199e..bdfe796ff 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/paired.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/paired.rs @@ -5,6 +5,7 @@ use cyber_server::runtime::{ }; struct Owner { + checkouts: checkouts::Checkouts, origin: SnapshotOrigin, journal: Dir, intent: MetadataPin, @@ -27,11 +28,7 @@ pub(super) fn eligible(record: &Journal) -> bool { && record.resolution.is_some() && record.settled_by.is_none() && record.origin.is_some() - && record - .context - .as_ref() - .is_some_and(|context| context.git.is_none()) - && record.backend != Some(JournalBackend::Git) + && record.context.is_some() } pub(super) fn reclaim( store: Store, @@ -71,24 +68,24 @@ fn owner( store: &Store, workers: &Dir, origin: &SnapshotOrigin, - checkout: &SnapshotRestoreCheckout, + evidence: (&SnapshotRestoreCheckout, &str), journal: Dir, digest: String, admitted: bool, ) -> Result { + let (checkout, receipt_digest) = evidence; store.verify_worker_namespace(origin)?; if store.worker_digest(Some(origin))?.as_ref() != Some(&digest) { return Err("Recovery cleanup intent differs".into()); } let lifetime = store.lock_lifetime(&checkout.lifetime)?; - let resources = lifetime - .checkout_bindings()? - .ok_or("Missing recovery checkout inventory")?; - if !resources.is_empty() - || checkout.resources_digest != checkout.lifetime.checkout_resources_digest(&resources)? - { - return Err("Managed recovery cleanup requires retained resource guards".into()); - } + let checkouts = checkouts::Checkouts::acquire( + store, + &lifetime, + &checkout.lifetime, + &checkout.resources_digest, + receipt_digest, + )?; let name = checkout .lifetime .source_phase @@ -104,6 +101,7 @@ fn owner( phase.verify(workers, name)?; let intent = MetadataPin::open(&journal, "intent.json")?; Ok(Owner { + checkouts, origin: origin.clone(), journal, intent, @@ -144,20 +142,17 @@ fn prepare( if !eligible(&record) || parent.origin.as_ref() != Some(&receipt.original) || parent.settled_by.as_deref() != Some(receipt.resolution_id.as_str()) - || parent - .context - .as_ref() - .is_none_or(|context| context.git.is_some()) + || parent.context.is_none() { return Err("Recovery cleanup pair or backend changed".into()); } - unmanaged(&store)?; + let receipt_digest = receipt.checkout_acknowledgement_digest()?; let workers = storage::existing_child(&store.namespace, "workers", true)?; let original = owner( &store, &workers, &receipt.original, - original_checkout, + (original_checkout, &receipt_digest), parent_dir, resolution.original_digest.clone(), true, @@ -166,7 +161,7 @@ fn prepare( &store, &workers, &checkout.resolution_origin, - &checkout.resolution, + (&checkout.resolution, &receipt_digest), journal, receipt .resolution_digest @@ -324,8 +319,8 @@ impl PreparedPair { if !due(self.config.as_ref(), &self.store.root, self.settled_at_ms) { return Err("Recovery retention no longer permits cleanup".into()); } - unmanaged(&self.store)?; for owner in &self.owners { + owner.checkouts.verify(&self.store)?; owner.lifetime.verify()?; self.store.verify_worker_namespace(&owner.origin)?; owner.intent.verify(&owner.journal, "intent.json")?; diff --git a/crates/cyber-snapshot/src/git/prepared.rs b/crates/cyber-snapshot/src/git/prepared.rs index 102dba20d..ecdd6dbaf 100644 --- a/crates/cyber-snapshot/src/git/prepared.rs +++ b/crates/cyber-snapshot/src/git/prepared.rs @@ -185,7 +185,11 @@ impl Shadow { }) .transpose()?; let mut remaining = max_total; - let mut trees = Vec::new(); + let mut trees: Vec = Vec::new(); + // References into already charged images avoid another byte cache and + // repeated helper launches for the same immutable blob. Every placement + // still consumes its full handoff budget and is hash-verified below. + let mut blobs = BTreeMap::new(); for (index, id) in ids.into_iter().enumerate() { let mut entries = BTreeMap::new(); if ImageSource::for_id(id)? == ImageSource::Copy { @@ -213,14 +217,18 @@ impl Shadow { let limit = usize::try_from(max_file.min(remaining).min(64 * 1024 * 1024)) .map_err(|_| "Git journal image limit overflow")?; let bytes = self - .run_bounded(&["cat-file", "blob", object], limit) + .restore_blob(object, &blobs, &trees, &entries, limit) .await?; images::verify(object, &bytes)?; remaining = remaining .checked_sub(bytes.len() as u64) .ok_or("Git journal image handoff exceeds limit")?; + let relative = relative(prefix, path)?; + blobs + .entry(object.clone()) + .or_insert_with(|| (trees.len(), relative.clone())); entries.insert( - relative(prefix, path)?, + relative, Image { source: ImageSource::Git, mode: mode.clone(), @@ -251,6 +259,29 @@ impl Shadow { }) } + async fn restore_blob( + &self, + object: &str, + blobs: &BTreeMap, + trees: &[PreparedTree], + current: &BTreeMap, + limit: usize, + ) -> Result, String> { + let Some((tree, path)) = blobs.get(object) else { + return self.run_bounded(&["cat-file", "blob", object], limit).await; + }; + let images = if *tree == trees.len() { + current + } else { + &trees.get(*tree).ok_or("Missing prepared Git tree")?.images + }; + let image = images.get(path).ok_or("Missing prepared Git blob")?; + if image.bytes.len() > limit { + return Err("Git journal image handoff exceeds limit".into()); + } + Ok(image.bytes.clone()) + } + async fn git_image_exclusions( &self, ids: [&String; 3], diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 5c0f7ea22..6b97935f9 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -1,5 +1,74 @@ # Active implementation goal and delivery rules +## Current audit: 2026-11-09 + +The user has set the active objective to complete implementation, verification +and accepted delivery of **#93**. #92 remains a separate unfinished workstream; +preserve its accepted behavior and dependency gates without assuming its missing +guarantees or selecting its pending policies. The owner's closed #94 is outside +this objective. The #92 rows below retain dependency context from the earlier +combined audit; they are not additional scope for the current goal. + +Main `8266db6c` is integrated into `feat/managed-recovery-owner` at `2fcd745d`. +The merge retains snapshot admission/reclamation and the incoming root-schema +validation/settlement events, plus the binary-worker activity-drain wait. The +seven pre-existing uncommitted files were restored from a retained backup stash. +The branch still contains substantial undelivered recovery work; it must not be +confused with accepted main behavior. GitHub reported both issues open and no +open PRs at audit time. The issue #93 checklist trails the repository's merged +section 5/6 implementation state; reconcile it at delivery. + +| Area | Evidence and remaining work | +| --- | --- | +| #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill acceptance passes locally, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | +| #93 rules/context, tasks 4.1–4.4 | Implement trusted path-rule discovery/precedence/bounds and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 structured results, tasks 5.1–5.4 | #116 merged admission/schema bounds, finalization, retries, durable per-input outcomes and CLI/API/SDK rendering. Three real CLI and five server tests pass after integration. Compaction during an open attempt and budget/interrupt terminal outcomes still need acceptance; do not infer them from restart between completed prompts. | +| #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | +| #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | +| #92 protocol/callbacks/producers, tasks 3–6/S/D | Finish contribution ownership, hooks/providers/catalog transforms and TypeScript kit, durable scoped callbacks (`session/prompt` still explicitly refuses), Setup and DirectoryAdded plus combined producer acceptance. Preserve accepted FileChanged and CwdChanged fixes. Complete S1 epoch/directory-key/legacy-deletion and D1 directory-admission/Safe Boundary/child-policy review before their dependent code. | + +### Focused code-review findings and next actions + +This review covers incoming JSONL behavior and merge overlap, not acceptance of +the entire recovery branch. Rust integration tests are executable; native +cross-platform, SDK/plugin and full-workspace checks have not been rerun here. + +1. **Integration regression: optional snapshot capture fails JSONL results.** + Unchanged main passes all six JSONL tests. The integrated candidate initially + fails four because `Driver::failed` treats `kind: snapshot` as a failed input, + despite successful inference and ordinary exec's nonfatal warning handling. + The local fix keeps these diagnostics nonfatal and adds a real-binary + regression with unavailable Git snapshots. All seven JSONL tests and all + three structured-output CLI tests pass after the fix. +2. **Major: request and interruption ownership is Session-wide.** + `Driver::asked` accepts every matching Session request and `interrupt` targets + the Session without binding to the driver's admitted messages. Track exact + input/request authority and test two drivers/owner clients sharing a Session, + including unrelated work present before driver startup. +3. **Major: long runs retain completed payloads and output blocks the loop.** + `inputs` retains every completed text/structured result; the pending-input cap + does not bound that history. `Out::line` writes/flushes stdout synchronously + on the same loop that handles controls and lease renewal. Retire emitted + payloads into bounded accounting and use a bounded output owner with explicit + I/O-failure settlement; test long runs, slow readers and broken pipes. +4. **Major: last-message publication uses a predictable temporary path.** + `write_atomically` truncates `..tmp` without exclusive creation, allowing + concurrent writers or a pre-created symlink to redirect/interfere with the + write. Use an exclusively created temporary file beside the destination and + test symlink pre-creation, concurrent writers and cleanup after failure. + +Verification: three structured-output CLI tests, seven JSONL CLI tests, five +structured-result server tests and six binary-worker ownership tests pass; +formatting and whitespace +checks pass. CI-pinned OpenSpec 1.13.2 validates 100 items after the incoming +archives; spec lint reports zero errors and 25 warnings. The preparation case now +passes with an independent staging deadline. A separate unconstrained-run +first-prompt startup stall remains unresolved; its isolated rerun and the complete +suite with four parallel threads pass. The stalled fixture's diagnostics are +retained rather than counted as a passing run. +The integration regression is locally corrected; finish the current recovery +delivery increment before starting another broad implementation stream. + The full approved scope from PRs #53/#54 remains required. Its original delivery order was: 1. [Plugins and hooks](../openspec/changes/complete-plugin-hook-runtime/tasks.md): diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 821b8cab5..f9e301e98 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -159,6 +159,31 @@ finish acknowledged payload reclamation but must not delete unacknowledged evide or replay filesystem resolution. Unknown files, aliases and identity changes refuse cleanup visibly. No direct user removal of private journal directories is required. +## Managed checkout payload reclamation + +Reclamation of acknowledged regular-file Git journal payloads SHALL reacquire +the complete persisted checkout set for each consumed worker lifetime before +preparing a durable deletion plan. Each claim SHALL bind the exact original +creation and lifetime, exclude a live worker, and retain an already published +checkout acknowledgement matching both the resource digest and the complete +consuming receipt digest. Synchronize those exact retained markers and their +directories before consuming them, covering publisher death after rename but +before its final sync. Cleanup SHALL NOT create or repair a missing marker. + +Retain all original/resolution native claim and acknowledgement pins through +the serialized writer callback and every private unlink. Revalidate the current +complete managed-location inventory and registrations, retained claim routes, +journal ownership and retention policy before deletion. An absent/foreign marker, +live claim, changed registration or incomplete inventory SHALL refuse cleanup +before publishing a plan. Source contents, Git index/configuration and original +native evidence SHALL remain unchanged. Restart after a durable plan SHALL use +the same frozen payload identities and complete-resource proof, never source +replay. Existing receipt and reclamation event formats remain unchanged. + +Typed link payloads, incomplete construction, legacy missing proof, CAS/metadata +collection and the complete platform/cut matrix remain separate obligations. +This extension does not enable default owned Git restore admission. + ## Compatibility and migration Historical bare Git trees remain readable with explicit unknown exclusion history. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index ead65d466..2b2576deb 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -463,3 +463,23 @@ - **WHEN** native checkout admission reads Session policy during an owned restore or recovery - **THEN** the caller SHALL release the Session projection mutex before that host call and retain an immutable captured revision - **AND** a changed revision SHALL refuse worker admission or subsequent source effects, preserving existing native evidence + +#### Scenario: Repeated immutable Git blobs within a native handoff +- **WHEN** multiple paths or target/recorded/expected images refer to one verified Git blob +- **THEN** the handoff MAY reuse its verified bytes without retaining another payload cache +- **AND** each image SHALL retain its own path/mode, verify its object digest and consume its full per-file/combined image budget +- **AND** reuse SHALL remain local to that handoff; a later handoff SHALL detect corrupt object storage rather than using earlier bytes + +#### Scenario: Managed Git recovery after actual partial replacement and process death +- **GIVEN** the real CLI/service has replaced some managed Git source files while other files retain their pre-restore images +- **WHEN** the service is killed and restarted and the owner confirms keep or rollback against the native review +- **THEN** keep SHALL preserve the observed partial namespace and later owner edits, and rollback SHALL restore proven displaced preimages +- **AND** both SHALL retain exact original/resolution checkout resource receipts, preserve unrelated owner files and the transcript, and leave user Git index/configuration unchanged +- **AND** repeated request lookup and explicit metadata acknowledgement after restart SHALL NOT replay source changes or start inference + +#### Scenario: Reclaiming acknowledged managed Git payloads +- **GIVEN** an aged acknowledged Git restore or recovery pair has exact committed receipts, persisted checkout sets and regular-file payload ownership +- **WHEN** maintenance prepares and consumes a durable reclamation plan +- **THEN** it SHALL retain every exited checkout claim and already published acknowledgement matching the exact lifetime, resource digest and consuming receipt through deletion +- **AND** a missing acknowledgement, live claim or changed registration SHALL refuse before plan publication without repairing metadata +- **AND** restart after plan publication SHALL use frozen native payload identities without source replay, preserving original claims, receipts and user Git metadata diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 54ededc8a..f7b579d18 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -142,7 +142,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. ## 6. Streaming JSONL input - [x] 6.1 Define typed version-2 envelopes, correlation, resource bounds, aggregate exit rules and driver-owned interactive request lifetime; document compatibility and unsupported combinations. -- [x] 6.2 Implement concurrent bounded parsing/admission and exact owned permission/question replies, retained request lookup, interrupt and end/EOF draining without stopping unrelated work. +- [ ] 6.2 Implement concurrent bounded parsing/admission and exact owned permission/question replies, retained request lookup, interrupt and end/EOF draining without stopping unrelated work. Basic parsing/admission merged in #117. The 2026-11-09 code review found that request collection and interrupt target the Session without checking the driver's admitted message ownership, and completed inputs retain their full result payloads indefinitely. Complete those ownership/resource bounds and their shared-Session/long-run regressions before accepting this task. - [ ] 6.3 Emit per-input results, optional partial/hook events and atomic last-message output with bounded backpressure and secret-safe errors. `--include-hook-events` is not implemented: hook receipts are deliberately not on the Session stream, so the lines would have nothing reliable to carry. - [ ] 6.4 Test multiple inputs, steering/queueing, malformed/oversized input, capacity, stale/cross-request replies, EOF while awaiting input, slow stdout, SIGINT, server restart and lost admission acknowledgement. Done: multiple inputs in order, per-input structured results, malformed input (exit 2), unknown request replies, EOF while a request waits, unsupported combinations, with the real binary (`cyber-cli/tests/exec_stream_input.rs`). Open: steering, capacity, slow stdout, SIGINT, server restart and lost admission acknowledgement. @@ -613,3 +613,81 @@ reopened metadata acknowledgement without source replay. This is an ordinary managed restore positive, not complete managed keep/rollback/process-death, construction/reclamation/platform acceptance. Default Git admission remains off; M2–M8 and all parent tasks remain open. + +Managed Git CLI process-death continuation: the explicit native-git-acceptance +binary creates a ready managed linked checkout in default application storage, +observes a genuinely partial namespace during a 1,280-file restore and kills the +registered service. Fresh services review/confirm keep and rollback, consume +non-empty original/resolution checkout bindings and reconcile metadata without +source replay. Verify all bulk file images, an early deletion, unrelated/later +owner files, escaped newline paths, stale/foreign requests, unchanged transcript +and user index/configuration, and no inference during recovery. Reuse verified +blob bytes only within one bounded image handoff while charging/verifying every +placement; SHA-1/SHA-256 tests retain mode, byte-limit and later-corruption refusal. +The four native workflow jobs now select this candidate, but workflow edits are +not execution evidence. These two source-kill cuts do not close the full cut, +resource construction, reclamation or platform matrix; default Git admission and +all parent acceptance remain unchanged. + +Resume verification (2026-11-08, macOS ARM): the repeated-blob regression passes +for SHA-1/SHA-256, snapshot all-target Clippy with `-D warnings` passes, and +format/whitespace checks pass. The candidate CLI suite reports 19 passes and one +preparation timeout in +`fully_prepared_pre_effect_cancellation_reclaims_only_persisted_target_inodes`; +that case passes alone (60.54 seconds). Managed Git keep/rollback passes, now +including an explicit assertion for the newline-containing path. This is not a +green complete-suite result. Strict validation with CI-pinned OpenSpec 1.13.2 +passes all 117 items; spec lint reports zero errors and 25 warnings. The default +CLI run was stopped before execution; native CI/platform runs remain unperformed. +Fetched main is now `8266db6c`, including merged structured-output PR #116 and +stream-input PR #117. Integrate and audit their remaining acceptance obligations +before continuing sections 5/6; both issues remain open. These observations do +not close parent tasks or establish the preparation timeout's origin. + +Main integration audit (2026-11-09): main `8266db6c` is merged at `2fcd745d`, +retaining the recovery branch and all seven uncommitted files. Incoming #116/#117 +are now included; task 6.2 is reopened for exact driver ownership and bounded +retained state. Unchanged main passes its six JSONL tests. The integrated branch +initially fails four because optional snapshot warnings are treated as failed +inputs. After matching ordinary exec's nonfatal warning handling and adding a +real-binary regression, all seven JSONL and three structured-output CLI tests +pass. Five structured-result server and six binary-worker ownership tests also +pass, as do formatting/whitespace checks and strict OpenSpec (100 items after +incoming archives; lint zero errors/25 warnings). The focused review and remaining +scope are recorded in docs/goal-delivery.md. Full workspace/SDK/plugin/native CI +were not rerun; the prior preparation timeout and all parent acceptance gates +remain open. No goal implementation PR is open and no issue was closed. + +Managed regular-file Git payload reclamation continuation (2026-11-09): ordinary +restores and both recovery owners now reacquire their complete persisted checkout +sets, require the exact already published consuming acknowledgements, synchronize +those retained markers/directories and hold all native pins through private +deletion. Missing markers never create metadata or a reclamation plan. Live +claims, changed registrations/resource digests and foreign dispositions refuse. +The existing plan/receipt formats are retained; data-only validation no longer +mistakes a managed resource set for an unsupported shape. Native phase checks +still require exact admission/origin and resource-digest evidence. Git restore +default admission remains off; typed link payloads, incomplete construction, +CAS/metadata collection and the full native/cut matrix remain open with M2–M8. + +The real managed CLI keep/rollback cases now remove an acknowledgement and hold a +live claim to prove precise pre-plan refusals, then interrupt after durable plan +publication and resume reclamation without source replay. The App verifies aged +ordinary managed cleanup, reopened receipt lookup, preserved later owner symlink, +no inference and unchanged user index/configuration. The core read-only marker +requirement and persisted source-phase/digest cases pass. The native workflow +selects the new core requirement on all four platforms; edits are not CI evidence. + +Local checks: all 20 candidate CLI branch/recovery tests pass with four parallel +threads; both Git App acceptance tests pass. Snapshot library: 140 passed and six +ignored subprocess fixtures; focused final marker and source-phase tests pass. +All-target Clippy for core/snapshot/server/App/CLI passes with warnings denied, +as do formatting/whitespace and strict OpenSpec (100 items; lint zero errors and +25 warnings). The preparation kill helper now gives staging an independent +bounded deadline; the service-kill probe matches the recorded server instance +rather than any listener at a reused port. An unconstrained parallel run passed +19 cases but stalled before conversation-test prompt admission; that attempt +was terminated after preserving its Store/registration and samples under +/tmp/cyber-goal-first-prompt-stall-20261109. Its isolated rerun passed. That run +is not green and its startup-stall cause remains unassigned. SDK/plugin/full +workspace and external native CI have not been rerun for this checkpoint. From 351fd1e037603795208c92636967a6ccced73738 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 16:24:58 -0400 Subject: [PATCH 52/87] Reclaim acknowledged symbolic-link journal payloads --- .github/workflows/non-git-snapshot-proof.yml | 6 + .../cyber-app/tests/snapshot_git_runtime.rs | 56 +++++++- crates/cyber-cli/tests/branches_cli.rs | 131 +++++++++++++----- crates/cyber-snapshot/src/copy/reclamation.rs | 8 +- .../src/copy/reclamation/paired.rs | 14 +- .../managed-recovery.md | 14 ++ .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 31 +++++ 8 files changed, 218 insertions(+), 49 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 4a3141ca7..53bfc0362 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -45,6 +45,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: key: non-git-snapshot-proof + - name: Sandbox prerequisites + if: runner.os == 'Linux' + run: | + sudo rm -f /etc/apt/sources.list.d/microsoft-prod.* && sudo apt-get update && sudo apt-get install -y bubblewrap + # Ubuntu 24.04 restricts the user namespaces required by bubblewrap. + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true - name: Native storage, conflict and process-kill proof shell: bash run: | diff --git a/crates/cyber-app/tests/snapshot_git_runtime.rs b/crates/cyber-app/tests/snapshot_git_runtime.rs index cda5f9029..6794f7b41 100644 --- a/crates/cyber-app/tests/snapshot_git_runtime.rs +++ b/crates/cyber-app/tests/snapshot_git_runtime.rs @@ -283,7 +283,7 @@ async fn managed_worktree_in_default_data_directory_restores_through_owned_worke config ); reconcile_after_owner_edit(&f, &receipt).await; - reclaim_managed_restore(&f, &receipt).await; + reclaim_restore_payloads(&f, &receipt).await; assert_eq!(std::fs::read(&index_path).unwrap(), index); assert_eq!( std::fs::read(f.project.join(".git/config")).unwrap(), @@ -292,7 +292,7 @@ async fn managed_worktree_in_default_data_directory_restores_through_owned_worke f.app.runtime.shutdown().await; } -async fn reclaim_managed_restore(f: &Fixture, receipt: &SnapshotRestoreReceipt) { +async fn reclaim_restore_payloads(f: &Fixture, receipt: &SnapshotRestoreReceipt) { use cyber_server::runtime::snapshot_reclamation::Receipt; let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); let snapshots = cyber_snapshot::GitSnapshots::with_reclamation( @@ -327,6 +327,7 @@ async fn reclaim_managed_restore(f: &Fixture, receipt: &SnapshotRestoreReceipt) Ok(()) }) .unwrap(); + assert_replaced_link_payload_refused(f, receipt, &snapshots, policy.as_ref()).await; assert!( snapshots .reclaim(f.directory.to_str().unwrap()) @@ -351,6 +352,47 @@ async fn reclaim_managed_restore(f: &Fixture, receipt: &SnapshotRestoreReceipt) assert_reclaimed_source_and_receipt(f, receipt, inference).await; } +async fn assert_replaced_link_payload_refused( + f: &Fixture, + receipt: &SnapshotRestoreReceipt, + snapshots: &cyber_snapshot::GitSnapshots, + policy: &dyn cyber_snapshot::reclamation::Policy, +) { + use cyber_server::runtime::snapshot_reclamation::Receipt; + let journal = f.journal(receipt); + let intent: Value = + serde_json::from_slice(&std::fs::read(journal.join("intent.json")).unwrap()).unwrap(); + let slot = intent["changes"] + .as_array() + .unwrap() + .iter() + .position(|change| change["before"]["kind"] == "symlink"); + let Some(slot) = slot else { return }; + let path = journal.join(format!("stage_{slot}")); + let target = std::fs::read_link(&path).unwrap(); + assert!( + !path.exists(), + "journal link target must be dangling in this fixture" + ); + let saved = f._temp.path().join("retained-link-payload"); + std::fs::rename(&path, &saved).unwrap(); + std::os::unix::fs::symlink(&target, &path).unwrap(); + let error = snapshots + .reclaim(f.directory.to_str().unwrap()) + .await + .unwrap_err(); + assert!(error.contains("original inode changed"), "{error}"); + assert!( + policy + .record(&Receipt::Restore(receipt.clone())) + .unwrap() + .is_none() + ); + assert_eq!(std::fs::read_link(&path).unwrap(), target); + std::fs::remove_file(&path).unwrap(); + std::fs::rename(&saved, &path).unwrap(); +} + async fn assert_reclaimed_source_and_receipt( f: &Fixture, receipt: &SnapshotRestoreReceipt, @@ -436,6 +478,16 @@ async fn actual_git_worker_commits_receipt_and_reconciles_without_source_replay( assert_eq!(intent["acknowledged"], true); assert_eq!(receipt.origin.binding.directory, f.directory); reconcile_after_owner_edit(&f, &receipt).await; + let intent: Value = + serde_json::from_slice(&std::fs::read(journal.join("intent.json")).unwrap()).unwrap(); + assert!( + intent["changes"] + .as_array() + .unwrap() + .iter() + .any(|change| change["before"]["kind"] == "symlink") + ); + reclaim_restore_payloads(&f, &receipt).await; f.app.runtime.shutdown().await; } } diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 4f3a87c41..27e867b1c 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -867,6 +867,7 @@ fn managed_git_partial_kill(action: &str) { std::fs::write(env.directory.join(format!("bulk_{index:04}")), "before").unwrap(); } std::fs::write(env.directory.join("zz\nfixture"), "before").unwrap(); + std::os::unix::fs::symlink("bulk_0000", env.directory.join("a_link")).unwrap(); git_fixture(&env.directory, &["add", "--all"]); let index_path = PathBuf::from(git_fixture( &env.directory, @@ -881,6 +882,8 @@ fn managed_git_partial_kill(action: &str) { std::fs::write(env.directory.join(format!("bulk_{index:04}")), "after").unwrap(); } std::fs::write(env.directory.join("zz\nfixture"), "after").unwrap(); + std::fs::remove_file(env.directory.join("a_link")).unwrap(); + std::os::unix::fs::symlink("bulk_0001", env.directory.join("a_link")).unwrap(); std::fs::write(env.directory.join("aa_removed"), "after-only source").unwrap(); env.exec(Some(&id), "after snapshot"); let messages = kill_partial_restore(&env, &id, &first); @@ -942,39 +945,13 @@ fn assert_managed_git_reclamation(env: &Env, result: &Value) { serde_json::from_value(phase["checkout_bindings"].clone()).unwrap(); assert!(!bindings.is_empty()); let binding = &bindings[0]; - let marker = binding - .common_dir - .join("cyber-worktree-activity") - .join(&binding.worktree_id) - .join(format!("{}.snapshot-ack", binding.owner_id)); - let hidden = env.root.join("retained-checkout-marker"); let source = env.file("bulk_0000"); + let source_link = std::fs::read_link(env.directory.join("a_link")).unwrap(); assert_eq!(maintenance.reclaim(&env.directory).unwrap(), 0); maintenance.age(checkout.resolution_origin.activity_id.clone()); let resources = original.lifetime.checkout_owner().unwrap(); let receipt = Receipt::Recovery(receipt.clone()); - std::fs::rename(&marker, &hidden).unwrap(); - let refused = maintenance.reclaim(&env.directory).unwrap_err(); - assert!( - refused.contains("acknowledgement has not been published"), - "{refused}" - ); - assert!( - !marker.exists(), - "cleanup must not publish a missing marker" - ); - assert!(maintenance.policy.record(&receipt).unwrap().is_none()); - std::fs::rename(&hidden, &marker).unwrap(); - let locations = - cyber_core::worktrees::Repository::managed_locations_at(&env.directory).unwrap(); - let live = locations[0] - .0 - .reacquire_snapshot(&locations[0].1, binding, &resources) - .unwrap(); - let refused = maintenance.reclaim(&env.directory).unwrap_err(); - assert!(refused.contains("worker is still active"), "{refused}"); - assert!(maintenance.policy.record(&receipt).unwrap().is_none()); - drop(live); + let marker = assert_managed_cleanup_refusals(env, &maintenance, &receipt, binding, &resources); let cut = maintenance.plan_only(&env.directory).unwrap_err(); assert!( cut.contains("fixture cut after durable paired plan"), @@ -999,14 +976,68 @@ fn assert_managed_git_reclamation(env: &Env, result: &Value) { ); assert_eq!(maintenance.reclaim(&env.directory).unwrap(), 0); assert_eq!(env.file("bulk_0000"), source); + assert_eq!( + std::fs::read_link(env.directory.join("a_link")).unwrap(), + source_link + ); assert!(marker.exists()); } +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn assert_managed_cleanup_refusals( + env: &Env, + maintenance: &reclamation_support::Maintenance, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, + binding: &cyber_core::worktrees::CheckoutBinding, + resources: &cyber_core::worktrees::CheckoutOwner, +) -> PathBuf { + let marker = binding + .common_dir + .join("cyber-worktree-activity") + .join(&binding.worktree_id) + .join(format!("{}.snapshot-ack", binding.owner_id)); + let hidden = env.root.join("retained-checkout-marker"); + std::fs::rename(&marker, &hidden).unwrap(); + let refused = maintenance.reclaim(&env.directory).unwrap_err(); + assert!( + refused.contains("acknowledgement has not been published"), + "{refused}" + ); + assert!( + !marker.exists(), + "cleanup must not publish a missing marker" + ); + assert!(maintenance.policy.record(receipt).unwrap().is_none()); + std::fs::rename(&hidden, &marker).unwrap(); + let locations = + cyber_core::worktrees::Repository::managed_locations_at(&env.directory).unwrap(); + let live = locations[0] + .0 + .reacquire_snapshot(&locations[0].1, binding, resources) + .unwrap(); + let refused = maintenance.reclaim(&env.directory).unwrap_err(); + assert!(refused.contains("worker is still active"), "{refused}"); + assert!(maintenance.policy.record(receipt).unwrap().is_none()); + drop(live); + marker +} + #[cfg(all( feature = "native-git-acceptance", any(target_os = "macos", target_os = "linux") ))] fn assert_managed_git_images(env: &Env, action: &str, partial: &[String]) { + assert_eq!( + std::fs::read_link(env.directory.join("a_link")).unwrap(), + PathBuf::from(if action == "rollback" { + "bulk_0001" + } else { + "bulk_0000" + }) + ); assert_eq!( env.file("bulk_0000").as_deref(), Some(if action == "rollback" { @@ -2202,15 +2233,18 @@ fn kill_partial_restore(env: &Env, id: &str, first: &str) -> Vec { .stderr(std::process::Stdio::piped()) .spawn() .unwrap(); - let deadline = std::time::Instant::now() + Duration::from_secs(45); + // Git materialization, native inventory checks and durable stage construction + // all precede the first source exchange, especially for mixed link/file images. + let deadline = std::time::Instant::now() + Duration::from_secs(120); while env.file("bulk_0000").as_deref() != Some("before") { - assert!( - pending.try_wait().unwrap().is_none(), - "restore exited before a source exchange" - ); + if pending.try_wait().unwrap().is_some() { + let output = pending.wait_with_output().unwrap(); + panic!("restore exited before a source exchange: {output:?}"); + } assert!( std::time::Instant::now() < deadline, - "source exchange was never observed" + "source exchange was never observed: {}", + restore_preparation_status(env) ); std::thread::sleep(Duration::from_millis(1)); } @@ -2236,6 +2270,35 @@ fn kill_partial_restore(env: &Env, id: &str, first: &str) -> Vec { messages } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn restore_preparation_status(env: &Env) -> String { + let root = env.root.join("cyber-home/data/snapshot/global"); + let mut states = Vec::new(); + for namespace in std::fs::read_dir(root).into_iter().flatten().flatten() { + let journals = namespace.path().join("journals"); + for journal in std::fs::read_dir(journals).into_iter().flatten().flatten() { + let path = journal.path(); + let intent = std::fs::read(path.join("intent.json")) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let stages = std::fs::read_dir(&path) + .into_iter() + .flatten() + .flatten() + .filter(|entry| entry.file_name().to_string_lossy().starts_with("stage_")) + .count(); + states.push(json!({ + "journal": journal.file_name().to_string_lossy(), + "stages": stages, + "changes": intent.as_ref().and_then(|v| v["changes"].as_array()).map(Vec::len), + "applied": intent.as_ref().map(|v| &v["applied"]), + "completed": intent.as_ref().map(|v| &v["completed"]), + })); + } + } + json!(states).to_string() +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_native_settlement_link(env: &Env, journal: &str, result: &Value) { let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index 876b691b0..9b5ed686c 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -239,10 +239,8 @@ fn prepare( .as_ref() .and_then(|files| files.get(name.as_str()).copied()); match (change.before.as_ref(), observed) { - (Some(before), Some(image)) - if before.kind == super::ImageKind::File && image.entry == *before => - { - let pin = storage::open(&journal, Path::new(&name))?; + (Some(before), Some(image)) if image.entry == *before => { + let pin = super::links::pin_image(&journal, std::ffi::OsStr::new(&name))?; ownership.verify_original(index, &pin)?; let file = Payload { journal: None, @@ -427,7 +425,7 @@ impl Prepared { Err(error) => return Err(storage::fail(error)), _ => {} } - let pin = storage::open(&self.journal, Path::new(&file.name))?; + let pin = super::links::pin_image(&self.journal, std::ffi::OsStr::new(&file.name))?; if storage::identity(&pin)? != file.identity || ownership::generation(&pin)? != file.generation { diff --git a/crates/cyber-snapshot/src/copy/reclamation/paired.rs b/crates/cyber-snapshot/src/copy/reclamation/paired.rs index bdfe796ff..96d265849 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/paired.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/paired.rs @@ -256,9 +256,8 @@ fn original_payloads( let file = expected .cloned() .map(|image| payload(owner, name.clone(), image)); - if expected.is_some_and(|image| image.entry.kind != super::super::ImageKind::File) - || (actual.as_ref() != expected - && !(actual.is_none() && file.as_ref() == planned.get(name.as_str()).copied())) + if actual.as_ref() != expected + && !(actual.is_none() && file.as_ref() == planned.get(name.as_str()).copied()) { return Err("Original recovery private slot differs from acknowledged outcome".into()); } @@ -287,10 +286,9 @@ fn resolution_payloads( ownership::observe_image(store, Some(&owner.journal), std::ffi::OsStr::new(&name))?; let previous = planned.get(name.as_str()).copied(); match (&change.before, actual) { - (Some(before), Some(image)) - if before.kind == super::super::ImageKind::File && image.entry == *before => - { - let pin = storage::open(&owner.journal, Path::new(&name))?; + (Some(before), Some(image)) if image.entry == *before => { + let pin = + super::super::links::pin_image(&owner.journal, std::ffi::OsStr::new(&name))?; ownership.verify_original(index, &pin)?; files.push(payload(owner, name, image)); } @@ -407,7 +405,7 @@ impl PreparedPair { Err(error) => return Err(storage::fail(error)), _ => {} } - let pin = storage::open(&owner.journal, Path::new(&file.name))?; + let pin = super::super::links::pin_image(&owner.journal, std::ffi::OsStr::new(&file.name))?; if storage::identity(&pin)? != file.identity || ownership::generation(&pin)? != file.generation { diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index f9e301e98..beea6c010 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -994,3 +994,17 @@ claims against enclosing registrations and exact activity/Session creation/Locat Unowned callers SHALL refuse. Session policy lookup during native admission SHALL run without the Session projection mutex held; immutable captured revisions and existing per-effect/consuming writer checks SHALL fence concurrent changes. + +### Typed acknowledged payload cleanup + +Ordinary and paired acknowledged cleanup MAY include immutable symbolic-link +images alongside regular images. The same journal inventory, original/prepared +ownership, receipt, reference, lifetime and retained checkout requirements apply. +Observe raw bounded link target bytes without following them, and pin each link +inode using the platform's no-follow primitive. Before every private unlink, +revalidate the frozen inode identity and metadata generation and retained parent +route. Equal target bytes on a replacement inode SHALL NOT grant cleanup authority. +Dangling journal targets are valid typed images; cleanup SHALL NOT dereference +them or consult current source paths. Missing payloads require the exact committed +plan for resumption. This extension grants no cleanup authority for incomplete +construction, unsupported leaf kinds, cancelled Git preparation or unowned links. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 2b2576deb..331c1e605 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -483,3 +483,10 @@ - **THEN** it SHALL retain every exited checkout claim and already published acknowledgement matching the exact lifetime, resource digest and consuming receipt through deletion - **AND** a missing acknowledgement, live claim or changed registration SHALL refuse before plan publication without repairing metadata - **AND** restart after plan publication SHALL use frozen native payload identities without source replay, preserving original claims, receipts and user Git metadata + +#### Scenario: Acknowledged private payload is a symbolic link +- **GIVEN** an ordinary restore or settled recovery pair has immutable typed link images and exact private inode ownership +- **WHEN** reference-aware maintenance prepares or resumes its receipt-bound cleanup plan +- **THEN** it SHALL pin and unlink the journal link inode without opening its target, including dangling targets +- **AND** an equal-target replacement inode or changed metadata SHALL refuse before deletion; a missing leaf SHALL require the exact previously committed plan +- **AND** later owner source links, consuming receipts and native ownership metadata SHALL remain intact diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index f7b579d18..0c1586132 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -691,3 +691,34 @@ was terminated after preserving its Store/registration and samples under /tmp/cyber-goal-first-prompt-stall-20261109. Its isolated rerun passed. That run is not green and its startup-stall cause remains unassigned. SDK/plugin/full workspace and external native CI have not been rerun for this checkpoint. + +Typed acknowledged-payload continuation (2026-11-09): ordinary and paired cleanup +now admits immutable symlink images through the existing no-follow native inode +pins, retaining the same complete preflight, frozen plan and per-unlink identity, +generation, route and checkout checks. App acceptance passes both managed restore +and SHA-1/SHA-256 root/scoped cases, including dangling journal targets and refusal +of equal-target replacement inodes before a plan exists. No source replay or +receipt/schema change is introduced. Cancelled Git preparation, incomplete +construction, CAS/metadata collection and parent acceptance remain open. Focused +code/security review found no additional issue in this extension; it is not a +review or acceptance of the whole branch. + +Native CI run 42684837811 on 3d3471f9 passed the snapshot adapter and exact core +marker test on Linux x64/ARM, then failed managed App worktree creation because +bubblewrap was not installed. The direct Git App worker case passed on both. +The workflow now installs bubblewrap and applies the same Ubuntu user-namespace +prerequisite as ci.yml. This workflow edit is not passing CI evidence; fresh +source-bound platform results are still required. + +Typed-payload local checks: snapshot library 140 passed, six ignored subprocess +fixtures; Git App acceptance two passed; exact managed Git real partial-kill +keep/rollback passed (315.50 seconds). The first expanded mixed-image CLI run +failed its 45-second first-exchange wait before reaching cleanup assertions. +The helper now permits 120 seconds for bounded preparation, reports journal stage +and intent state on timeout, and captures a prematurely exited command's output. +The same partial-window and post-kill source assertions remain required. This +retry does not establish the original timeout's precise cause or resolve the +separately preserved first-prompt startup stall. All-target Clippy for +core/snapshot/server/App/CLI passes with warnings denied. Formatting, whitespace, +strict OpenSpec (100 items) and lint (zero errors/25 warnings) pass. The complete +CLI suite was not repeated for this typed-payload extension. From 9ac06c40f6a557847a61ec5e12f12fdf22a0ea45 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 16:26:12 -0400 Subject: [PATCH 53/87] Publish JSONL last messages through exclusive temporary files --- .github/workflows/non-git-snapshot-proof.yml | 9 +++ crates/cyber-cli/src/commands/exec/driver.rs | 26 +++---- .../src/commands/exec/last_message.rs | 75 +++++++++++++++++++ crates/cyber-cli/src/commands/exec/mod.rs | 1 + crates/cyber-cli/tests/exec_stream_input.rs | 24 ++++++ docs/goal-delivery.md | 13 ++-- docs/local-core-guide.md | 2 +- .../specs/exec-mode/spec.md | 6 ++ .../complete-daily-coding-surfaces/tasks.md | 14 ++++ 9 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 crates/cyber-cli/src/commands/exec/last_message.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 53bfc0362..f6403240f 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -13,6 +13,7 @@ on: - crates/cyber-cli/Cargo.toml - crates/cyber-cli/src/** - crates/cyber-cli/tests/branches_cli.rs + - crates/cyber-cli/tests/exec_stream_input.rs - crates/cyber-cli/tests/support/** - crates/cyber-server/src/workspace.rs - crates/cyber-core/src/binary/native/** @@ -51,6 +52,7 @@ jobs: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.* && sudo apt-get update && sudo apt-get install -y bubblewrap # Ubuntu 24.04 restricts the user namespaces required by bubblewrap. sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + cargo build --locked -p cyber-sandbox --bin cyber-sandbox-exec - name: Native storage, conflict and process-kill proof shell: bash run: | @@ -72,6 +74,12 @@ jobs: run: | set -euo pipefail cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_git_keep_and_rollback_after_real_partial_restore_kill -- --nocapture 2>&1 | tee git-cli-recovery-proof.log + - name: JSONL driver and atomic last-message regressions + shell: bash + run: | + set -euo pipefail + cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::exec::last_message::tests -- --nocapture 2>&1 | tee jsonl-driver-proof.log + cargo test --locked -p cyber-cli --features native-git-acceptance --test exec_stream_input -- --nocapture 2>&1 | tee -a jsonl-driver-proof.log - name: Retain native evidence if: always() uses: actions/upload-artifact@v4 @@ -83,4 +91,5 @@ jobs: git-adapter-proof.log git-runtime-proof.log git-cli-recovery-proof.log + jsonl-driver-proof.log if-no-files-found: warn diff --git a/crates/cyber-cli/src/commands/exec/driver.rs b/crates/cyber-cli/src/commands/exec/driver.rs index e53b3eb58..54c65fd5b 100644 --- a/crates/cyber-cli/src/commands/exec/driver.rs +++ b/crates/cyber-cli/src/commands/exec/driver.rs @@ -10,7 +10,6 @@ //! process exits once its inputs are answered and the Session is idle. use std::collections::BTreeMap; -use std::path::Path; use std::time::{Duration, Instant}; use cyber_client::{Client, Event}; @@ -571,7 +570,7 @@ impl Driver<'_> { self.emit_result(input); } } - let code = if self.interrupted { + let mut code = if self.interrupted { EXIT_INTERRUPTED } else if self.fatal.is_some() { EXIT_PROTOCOL @@ -583,16 +582,22 @@ impl Driver<'_> { .or(self.run.error.as_ref().map(|_| EXIT_ERROR)) .unwrap_or(0) }; + let mut final_error = self.fatal.or(self.run.error.clone()); if let Some(path) = &self.args.output_last_message - && let Err(error) = write_atomically(path, &self.last_text) + && let Err(error) = super::last_message::write(path, &self.last_text) { - self.out.error(&format!("--output-last-message: {error}")); + let message = format!("--output-last-message: {error}"); + self.out.error(&message); + final_error.get_or_insert(message); + if code == 0 { + code = EXIT_ERROR; + } } self.out.stream_line( "end", json!({ "exit_code": code, - "error": self.fatal.or(self.run.error.clone()), + "error": final_error, "inputs": self.inputs.len(), "usage": { "input": self.run.input_tokens, "output": self.run.output_tokens }, "cost_usd": self.run.combined_cost(), @@ -602,14 +607,3 @@ impl Driver<'_> { code } } - -/// Write `text` next to `path` and rename it over, so a reader never sees half a file. -fn write_atomically(path: &Path, text: &str) -> std::io::Result<()> { - let name = path - .file_name() - .map(|n| n.to_string_lossy().into_owned()) - .unwrap_or_default(); - let temporary = path.with_file_name(format!(".{name}.tmp")); - std::fs::write(&temporary, text)?; - std::fs::rename(&temporary, path) -} diff --git a/crates/cyber-cli/src/commands/exec/last_message.rs b/crates/cyber-cli/src/commands/exec/last_message.rs new file mode 100644 index 000000000..b163f76ac --- /dev/null +++ b/crates/cyber-cli/src/commands/exec/last_message.rs @@ -0,0 +1,75 @@ +//! Publish an answer without sharing temporary paths with other writers. +use std::io::{self, Write}; +use std::path::Path; + +pub(super) fn write(path: &Path, text: &str) -> io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let mut temporary = tempfile::Builder::new() + .prefix(".cyber-last-message-") + .tempfile_in(parent)?; + temporary.write_all(text.as_bytes())?; + temporary.as_file().sync_all()?; + temporary.persist(path).map_err(|error| error.error)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::write; + + #[cfg(unix)] + #[test] + fn existing_temporary_and_destination_links_do_not_redirect_writes() { + let dir = tempfile::tempdir().unwrap(); + let protected = dir.path().join("protected"); + std::fs::write(&protected, "owner bytes").unwrap(); + let path = dir.path().join("answer"); + let predictable = dir.path().join(".answer.tmp"); + std::os::unix::fs::symlink(&protected, &predictable).unwrap(); + std::os::unix::fs::symlink(&protected, &path).unwrap(); + write(&path, "answer bytes").unwrap(); + assert_eq!(std::fs::read_to_string(protected).unwrap(), "owner bytes"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "answer bytes"); + assert!(std::fs::symlink_metadata(predictable).unwrap().is_symlink()); + assert!(std::fs::symlink_metadata(path).unwrap().is_file()); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 3); + } + + #[test] + fn concurrent_writers_publish_one_complete_answer_and_remove_their_stages() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("answer"); + let answers: Vec<_> = (0..8) + .map(|i| format!("answer {i}\n").repeat(8192)) + .collect(); + let barrier = std::sync::Barrier::new(answers.len()); + std::thread::scope(|scope| { + for answer in &answers { + let (path, barrier) = (&path, &barrier); + scope.spawn(move || { + barrier.wait(); + write(path, answer).unwrap(); + }); + } + }); + assert!(answers.contains(&std::fs::read_to_string(path).unwrap())); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + } + + #[test] + fn failed_publication_preserves_the_destination_and_removes_its_stage() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("answer"); + std::fs::create_dir(&path).unwrap(); + std::fs::write(path.join("owner"), "preserved").unwrap(); + assert!(write(&path, "answer bytes").is_err()); + assert_eq!( + std::fs::read_to_string(path.join("owner")).unwrap(), + "preserved" + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + } +} diff --git a/crates/cyber-cli/src/commands/exec/mod.rs b/crates/cyber-cli/src/commands/exec/mod.rs index 2f72c9d43..76569c460 100644 --- a/crates/cyber-cli/src/commands/exec/mod.rs +++ b/crates/cyber-cli/src/commands/exec/mod.rs @@ -5,6 +5,7 @@ mod delegation; mod driver; mod frames; pub(crate) mod goals; +mod last_message; pub(crate) mod report; mod review_run; mod schema; diff --git a/crates/cyber-cli/tests/exec_stream_input.rs b/crates/cyber-cli/tests/exec_stream_input.rs index 8618609c3..991e98448 100644 --- a/crates/cyber-cli/tests/exec_stream_input.rs +++ b/crates/cyber-cli/tests/exec_stream_input.rs @@ -247,6 +247,30 @@ fn unavailable_snapshots_do_not_fail_a_successful_jsonl_input() { assert_eq!(run.exit(), Some(0)); } +#[test] +fn last_message_publication_failure_fails_the_run_without_losing_the_answer() { + let provider = Provider::start(&["text:completed answer"]); + let destination = tempfile::tempdir().unwrap(); + let path = destination.path().display().to_string(); + let mut run = Driven::start(&provider, &["--output-last-message", &path]); + let mut seen = Vec::new(); + run.send(r#"{"type":"user","text":"one"}"#); + run.send(r#"{"type":"end"}"#); + let result = run.until("result", &mut seen); + assert_eq!(result["result"], "completed answer"); + assert_eq!(result["exit_code"], 0); + let end = run.until("end", &mut seen); + assert_eq!(end["exit_code"], 1); + assert!( + end["error"] + .as_str() + .unwrap() + .contains("--output-last-message") + ); + assert_eq!(run.exit(), Some(1)); + assert_eq!(std::fs::read_dir(destination.path()).unwrap().count(), 0); +} + #[test] fn two_prompts_give_two_results_in_order_then_end() { let provider = Provider::start(&["text:first answer", "text:second answer"]); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 6b97935f9..98fea2d47 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -51,11 +51,14 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. on the same loop that handles controls and lease renewal. Retire emitted payloads into bounded accounting and use a bounded output owner with explicit I/O-failure settlement; test long runs, slow readers and broken pipes. -4. **Major: last-message publication uses a predictable temporary path.** - `write_atomically` truncates `..tmp` without exclusive creation, allowing - concurrent writers or a pre-created symlink to redirect/interfere with the - write. Use an exclusively created temporary file beside the destination and - test symlink pre-creation, concurrent writers and cleanup after failure. +4. **Fixed locally: last-message publication used a predictable temporary path.** + Publication now uses an exclusively created sibling file, synchronizes it and + atomically replaces the destination, with automatic temporary-file cleanup on + failure. Three unit regressions cover pre-created temporary/destination links, + concurrent complete answers and cleanup after failed publication. All eight + real JSONL tests pass, including failed publication preserving the answer while + returning aggregate exit 1 and an `end` error. Interrupted/protocol exit codes + retain precedence. This does not complete the ownership/backpressure findings. Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 16a39b76d..10d31f77e 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -653,7 +653,7 @@ cyber exec --output-schema findings.schema.json --format json "audit the depende - **Requests:** the driver holds an interactive lease on the Session while it reads input, so the Session asks it instead of denying, and the run starts in `default` mode unless `--mode` says otherwise. Only requests this Session raised can be answered; any other `request_id` is an `error` line with `code: protocol`, and the run goes on. Nothing is ever approved for you. - **Ending:** after `end` or EOF nothing new is admitted, requests still waiting are declined (`reject` for permissions, dismissed questions), and the process exits once every admitted prompt has its result and the Session is idle. - **Limits and errors:** a line is at most 1 MiB and at most 32 prompts may wait for an answer. A line that is not a valid envelope, an over-long line or a 33rd waiting prompt is a fatal protocol error: an `error` line, no more input, the Session interrupted, and exit 2. -- **Exit code:** 130 after SIGINT, 2 after a fatal protocol error, otherwise the first failed prompt's code in admission order (1), else 0. `--output-schema` applies to every prompt, whose `result` is then the validated object. +- **Exit code:** 130 after SIGINT, 2 after a fatal protocol error, otherwise the first failed prompt's code in admission order (1), else 0. Failure to publish `--output-last-message` makes an otherwise successful run exit 1 and reports the error in `end`; per-input results remain available. Publication uses an exclusively created sibling file, replaces a destination symlink without writing its target, and removes its temporary file on failure. `--output-schema` applies to every prompt, whose `result` is then the validated object. - **Not combined with:** a prompt argument, `--command`, `--goal`, `--timeout`, `--max-cost` or `--max-tokens` (exit 2 before any Session). `--include-hook-events` is not available yet. ## Budgets and deferred input diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md index a52f641a2..ba970694e 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md @@ -10,3 +10,9 @@ #### Scenario: EOF with a pending approval - **WHEN** the driver's input closes while one of its requests awaits a reply - **THEN** that request follows the defined disconnected/non-interactive policy and the process does not wait forever or fabricate approval + +#### Scenario: Last-message publication collides or fails +- **WHEN** a driver publishes its final assistant text to `--output-last-message` +- **THEN** it SHALL stage bytes in an exclusively created sibling file and atomically replace the destination without following a pre-existing temporary or destination link +- **AND** concurrent writers SHALL publish complete answers and clean up their own temporary files, including after publication failure +- **AND** a publication failure SHALL preserve per-input results, report the error in `end`, and fail an otherwise successful aggregate run without replacing an existing interruption or protocol exit code diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 0c1586132..04a12bcb9 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -722,3 +722,17 @@ separately preserved first-prompt startup stall. All-target Clippy for core/snapshot/server/App/CLI passes with warnings denied. Formatting, whitespace, strict OpenSpec (100 items) and lint (zero errors/25 warnings) pass. The complete CLI suite was not repeated for this typed-payload extension. + +JSONL last-message publication continuation (2026-11-09): exclusively created +same-directory temporary files replace the predictable `.name.tmp` path, with +synchronized bytes and cleanup after failed publication. Destination symlinks +are replaced without writing their targets. Concurrent writers publish one +complete answer. Publication failure preserves per-input results and produces +aggregate exit 1 plus an `end` error when the run would otherwise succeed; +interruption/protocol exit precedence is unchanged. Three unit and all eight +real JSONL regressions pass; CLI all-target Clippy passes with warnings denied. +Strict OpenSpec still passes all 100 items, lint zero errors/25 warnings. +The native workflow now selects these regressions and builds the Linux sandbox +helper after installing bubblewrap. External results for these edits are pending. +Task 6.3 remains open for hook events and bounded output ownership/backpressure; +6.2/6.4 and all parent acceptance obligations remain open. From 634a7ac65aae87134cc801b89612de92a328fc6e Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 16:37:07 -0400 Subject: [PATCH 54/87] Retire completed JSONL payloads into bounded accounting --- .../cyber-cli/src/commands/exec/background.rs | 50 +++++- crates/cyber-cli/src/commands/exec/driver.rs | 114 +++++------- crates/cyber-cli/src/commands/exec/inputs.rs | 163 ++++++++++++++++++ crates/cyber-cli/src/commands/exec/mod.rs | 1 + crates/cyber-cli/src/commands/exec/report.rs | 35 ++++ crates/cyber-cli/tests/exec_stream_input.rs | 28 +++ docs/goal-delivery.md | 17 +- .../specs/exec-mode/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 23 +++ 9 files changed, 350 insertions(+), 87 deletions(-) create mode 100644 crates/cyber-cli/src/commands/exec/inputs.rs diff --git a/crates/cyber-cli/src/commands/exec/background.rs b/crates/cyber-cli/src/commands/exec/background.rs index 4f7031477..cf40649aa 100644 --- a/crates/cyber-cli/src/commands/exec/background.rs +++ b/crates/cyber-cli/src/commands/exec/background.rs @@ -38,8 +38,8 @@ pub(super) struct Owned { baseline: HashSet, /// The forked command's delegation; `None` once known not to be forked. fork: Option, - /// Message IDs admitted to the parent since the run began. - admitted: HashSet, + /// Only the one fork's failed/cancelled handback needs live admission proof. + fork_handback_admitted: bool, last_seq: i64, lagged: Option<(i64, Instant)>, /// Jobs and the fork a stop already tried to cancel. @@ -62,7 +62,7 @@ impl Owned { session: session.into(), baseline, fork: None, - admitted: HashSet::new(), + fork_handback_admitted: false, last_seq: -1, lagged: None, stopped: HashSet::new(), @@ -78,7 +78,7 @@ impl Owned { session: session.into(), baseline: HashSet::new(), fork: None, - admitted: HashSet::new(), + fork_handback_admitted: false, last_seq: -1, lagged: None, stopped: HashSet::new(), @@ -91,6 +91,7 @@ impl Owned { pub fn command(&mut self, message: &str) -> Result<(), CliError> { let id = Runtime::skill_command_delegation_id(&self.session, message); self.fork = Some(Admission::existing(&self.session, &id)?); + self.fork_handback_admitted = false; Ok(()) } @@ -99,9 +100,11 @@ impl Owned { self.last_seq = self.last_seq.max(seq); } if event.kind.starts_with("session.prompt.admitted") - && let Some(id) = event.data["message_id"].as_str() + && event.session_id.as_deref() == Some(self.session.as_str()) + && let Some(fork) = &self.fork + && event.data["message_id"] == format!("msg_skill_handback_{}", fork.id) { - self.admitted.insert(id.into()); + self.fork_handback_admitted = true; } if event.kind == "session.error" && event.data["kind"] == "background_settlement" { self.settlement = event.data["message"].as_str().map(str::to_string); @@ -136,7 +139,6 @@ impl Owned { self.fork = None; return Ok(Settle::Done); }; - let handback = format!("msg_skill_handback_{}", fork.id); match record.status { DelegationStatus::Pending | DelegationStatus::Cancelling => Ok(Settle::Wait), DelegationStatus::Unknown => Err(CliError::runtime(format!( @@ -144,7 +146,7 @@ impl Owned { fork.id ))), DelegationStatus::Failed | DelegationStatus::Cancelled - if !stopping && !self.admitted.contains(&handback) => + if !stopping && !self.fork_handback_admitted => { let id = fork.id.clone(); self.handback_pending(&id).map(|()| Settle::Wait) @@ -341,3 +343,35 @@ fn append_error(run: &mut Run, message: String) { None => message, }); } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn only_the_exact_fork_handback_and_session_set_admission_evidence() { + let mut owned = Owned::empty("ses_1"); + owned.command("msg_1").unwrap(); + let expected = format!("msg_skill_handback_{}", owned.fork_id().unwrap()); + let mut event = Event { + kind: "session.prompt.admitted.1".into(), + data: json!({"message_id":expected}), + seq: Some(0), + session_id: Some("ses_other".into()), + }; + owned.observe(&event); + assert!(!owned.fork_handback_admitted); + event.session_id = Some("ses_1".into()); + for index in 0..4096 { + event.data = json!({"message_id":format!("msg_{index}")}); + owned.observe(&event); + } + assert!(!owned.fork_handback_admitted); + event.data = json!({"message_id":expected}); + owned.observe(&event); + assert!(owned.fork_handback_admitted); + owned.command("msg_2").unwrap(); + assert!(!owned.fork_handback_admitted); + } +} diff --git a/crates/cyber-cli/src/commands/exec/driver.rs b/crates/cyber-cli/src/commands/exec/driver.rs index 54c65fd5b..593a9b807 100644 --- a/crates/cyber-cli/src/commands/exec/driver.rs +++ b/crates/cyber-cli/src/commands/exec/driver.rs @@ -20,14 +20,13 @@ use tokio::sync::mpsc; use super::background::{Owned, Settle}; use super::frames::{self, Frame, MAX_LINE}; +use super::inputs::{Input, Inputs}; use super::report::{Out, Run}; use super::{EXIT_ERROR, EXIT_INTERRUPTED, ExecArgs, NO_APPROVER, api, safe_id}; use crate::error::CliError; /// How often the interactive lease (45 s) is renewed. const LEASE_RENEWAL: Duration = Duration::from_secs(15); -/// Inputs admitted and not yet answered, at most. -const MAX_PENDING: usize = 32; /// A fatal protocol error. const EXIT_PROTOCOL: u8 = 2; @@ -39,23 +38,6 @@ enum Line { Eof, } -/// One admitted input and its outcome. -struct Input { - /// The driver's correlation ID, when the line had one. - id: Option, - message_id: String, - answered: bool, - text: String, - error: Option, - structured: Option, -} - -impl Input { - fn exit_code(&self) -> u8 { - if self.error.is_some() { EXIT_ERROR } else { 0 } - } -} - /// A permission or question request this Session raised, with the Session to reply to. struct Ask { session_id: String, @@ -69,9 +51,9 @@ struct Driver<'a> { out: &'a mut Out, run: Run, owned: Owned, - inputs: Vec, - /// Indexes of promoted inputs whose answer is in progress. - answering: Vec, + inputs: Inputs, + /// Exact message identities of promoted inputs whose answer is in progress. + answering: Vec, /// The last finished step ended the answer (no tool calls followed). answer_done: bool, asks: BTreeMap, @@ -115,7 +97,7 @@ pub async fn execute( out: &mut out, run, owned, - inputs: Vec::new(), + inputs: Inputs::default(), answering: Vec::new(), answer_done: false, asks: BTreeMap::new(), @@ -234,13 +216,13 @@ impl Driver<'_> { /// Admit a prompt under a message ID chosen here; a lost acknowledgement is retried /// once with the same ID, which the server answers with the original receipt. async fn admit(&mut self, id: Option, parts: Vec, delivery: &str) { - let open = self.inputs.iter().filter(|i| !i.answered).count(); - if open >= MAX_PENDING { - self.protocol_error("more than 32 inputs waiting for an answer"); - self.fatal = Some("protocol error: input over capacity".into()); + let message_id = cyber_core::ids::new_id("msg"); + let input = Input::new(id, message_id.clone()); + if let Err(error) = self.inputs.admit(input) { + self.protocol_error(error); + self.fatal = Some(format!("protocol error: {error}")); return self.close().await; } - let message_id = cyber_core::ids::new_id("msg"); let mut body = json!({ "id": message_id, "parts": parts, "delivery": delivery }); if let Some(schema) = &self.args.schema { body["format"] = json!({ "type": "json_schema", "schema": schema }); @@ -250,14 +232,7 @@ impl Driver<'_> { if matches!(&sent, Err(cyber_client::ClientError::Transport(_))) { sent = self.client.post(&path, body).await; } - let mut input = Input { - id, - message_id, - answered: false, - text: String::new(), - error: None, - structured: None, - }; + let input = self.inputs.get_mut(&message_id).expect("reserved input"); match sent { Ok(_) => { self.idle = false; @@ -268,15 +243,14 @@ impl Driver<'_> { } Err(error) => { input.error = Some(error.to_string()); - input.answered = true; self.out.stream_line( "input", json!({ "id": input.id, "message_id": input.message_id, "status": "refused", "error": input.error }), ); + let input = self.inputs.complete(&message_id).expect("refused input"); self.emit_result(&input); } } - self.inputs.push(input); } async fn interrupt(&mut self) { @@ -387,8 +361,10 @@ impl Driver<'_> { "session.text.ended" => { self.out.durable(&mut self.run, kind, &event.data); self.last_text = self.run.text.clone(); - for &index in &self.answering { - self.inputs[index].text = self.run.text.clone(); + for message_id in &self.answering { + if let Some(input) = self.inputs.get_mut(message_id) { + input.text = self.run.text.clone(); + } } } "session.structured_output.settled" => self.structured(&event.data), @@ -428,27 +404,28 @@ impl Driver<'_> { /// A promoted input of this run joins the answer in progress, or starts the next one /// once the previous answer has ended. fn promoted(&mut self, data: &Value) { - let Some(index) = self - .inputs - .iter() - .position(|i| data["message_id"] == i.message_id.as_str()) - else { + let Some(message_id) = data["message_id"].as_str() else { return; }; + if !self.inputs.contains(message_id) { + return; + } if self.answer_done { self.flush_answers(); } + if !self.inputs.contains(message_id) || self.answering.iter().any(|id| id == message_id) { + return; + } self.answer_done = false; self.idle = false; self.run.text.clear(); - self.answering.push(index); + self.answering.push(message_id.to_owned()); } fn structured(&mut self, data: &Value) { - let Some(input) = self - .inputs - .iter_mut() - .find(|i| data["message_id"] == i.message_id.as_str()) + let Some(input) = data["message_id"] + .as_str() + .and_then(|id| self.inputs.get_mut(id)) else { return; }; @@ -462,10 +439,10 @@ impl Driver<'_> { fn failed(&mut self, data: &Value) { let message = data["message"].as_str().unwrap_or("error").to_owned(); self.out.error(&message); - for &index in &self.answering { - self.inputs[index] - .error - .get_or_insert_with(|| message.clone()); + for message_id in &self.answering { + if let Some(input) = self.inputs.get_mut(message_id) { + input.error.get_or_insert_with(|| message.clone()); + } } } @@ -501,11 +478,12 @@ impl Driver<'_> { /// Emit the results of the inputs whose answer ended. fn flush_answers(&mut self) { - for index in std::mem::take(&mut self.answering) { - self.inputs[index].answered = true; - let input = &self.inputs[index]; - self.emit_result(input); + for message_id in std::mem::take(&mut self.answering) { + if let Some(input) = self.inputs.complete(&message_id) { + self.emit_result(&input); + } } + self.run.retire_driver_answer(); self.answer_done = false; } @@ -539,7 +517,7 @@ impl Driver<'_> { return false; } // An input admitted but not yet promoted keeps the run open. - if self.inputs.iter().any(|i| !i.answered) && self.fatal.is_none() && !self.interrupted { + if !self.inputs.is_empty() && self.fatal.is_none() && !self.interrupted { return false; } match self @@ -559,16 +537,8 @@ impl Driver<'_> { /// The aggregate exit: 130 for SIGINT, 2 for a fatal protocol error, else the first /// failed input's code in admission order. Ends with one `end` line. fn finish(mut self) -> u8 { - for input in self.inputs.iter_mut().filter(|i| !i.answered) { - input - .error - .get_or_insert_with(|| "not answered before the run ended".into()); - input.answered = true; - } - for input in &self.inputs { - if input.error.as_deref() == Some("not answered before the run ended") { - self.emit_result(input); - } + for input in self.inputs.unfinished() { + self.emit_result(&input); } let mut code = if self.interrupted { EXIT_INTERRUPTED @@ -576,9 +546,7 @@ impl Driver<'_> { EXIT_PROTOCOL } else { self.inputs - .iter() - .map(Input::exit_code) - .find(|code| *code != 0) + .failure_code() .or(self.run.error.as_ref().map(|_| EXIT_ERROR)) .unwrap_or(0) }; @@ -598,7 +566,7 @@ impl Driver<'_> { json!({ "exit_code": code, "error": final_error, - "inputs": self.inputs.len(), + "inputs": self.inputs.total(), "usage": { "input": self.run.input_tokens, "output": self.run.output_tokens }, "cost_usd": self.run.combined_cost(), "duration_ms": self.run.started.elapsed().as_millis() as u64, diff --git a/crates/cyber-cli/src/commands/exec/inputs.rs b/crates/cyber-cli/src/commands/exec/inputs.rs new file mode 100644 index 000000000..91df95c87 --- /dev/null +++ b/crates/cyber-cli/src/commands/exec/inputs.rs @@ -0,0 +1,163 @@ +//! Keep only pending driver inputs; emitted answers leave bounded accounting. +use std::collections::BTreeMap; + +use serde_json::Value; + +use super::EXIT_ERROR; + +const MAX_PENDING: usize = 32; + +pub(super) struct Input { + ordinal: u64, + pub id: Option, + pub message_id: String, + pub text: String, + pub error: Option, + pub structured: Option, +} + +impl Input { + pub fn new(id: Option, message_id: String) -> Self { + Self { + ordinal: 0, + id, + message_id, + text: String::new(), + error: None, + structured: None, + } + } + + pub fn exit_code(&self) -> u8 { + if self.error.is_some() { EXIT_ERROR } else { 0 } + } +} + +#[derive(Default)] +pub(super) struct Inputs { + pending: BTreeMap, + total: u64, + failure: Option<(u64, u8)>, +} + +impl Inputs { + /// Reserve correlation before the HTTP request; refusal retires the same slot. + pub fn admit(&mut self, mut input: Input) -> Result<(), &'static str> { + if self.pending.len() >= MAX_PENDING { + return Err("more than 32 inputs waiting for an answer"); + } + if self.pending.contains_key(&input.message_id) { + return Err("duplicate driver message identity"); + } + let total = self.total.checked_add(1).ok_or("input count exhausted")?; + input.ordinal = self.total; + self.total = total; + self.pending.insert(input.message_id.clone(), input); + Ok(()) + } + + pub fn contains(&self, message_id: &str) -> bool { + self.pending.contains_key(message_id) + } + + pub fn get_mut(&mut self, message_id: &str) -> Option<&mut Input> { + self.pending.get_mut(message_id) + } + + /// The caller emits this value and drops it; no completed payload is retained. + pub fn complete(&mut self, message_id: &str) -> Option { + let input = self.pending.remove(message_id)?; + self.record_failure(&input); + Some(input) + } + + pub fn unfinished(&mut self) -> Vec { + let mut pending: Vec<_> = std::mem::take(&mut self.pending).into_values().collect(); + pending.sort_by_key(|input| input.ordinal); + for input in &mut pending { + input + .error + .get_or_insert_with(|| "not answered before the run ended".into()); + self.record_failure(input); + } + pending + } + + pub fn is_empty(&self) -> bool { + self.pending.is_empty() + } + + pub fn total(&self) -> u64 { + self.total + } + + pub fn failure_code(&self) -> Option { + self.failure.map(|(_, code)| code) + } + + fn record_failure(&mut self, input: &Input) { + let code = input.exit_code(); + if code != 0 + && self + .failure + .is_none_or(|(ordinal, _)| input.ordinal < ordinal) + { + self.failure = Some((input.ordinal, code)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input(id: usize) -> Input { + let mut input = Input::new(Some(id.to_string()), format!("msg_{id}")); + input.text = "answer".repeat(1024); + input.structured = Some(serde_json::json!({"answer":"structured"})); + input + } + + #[test] + fn completed_payloads_release_capacity_and_failures_survive_long_runs() { + let mut inputs = Inputs::default(); + for index in 0..4096 { + inputs.admit(input(index)).unwrap(); + if index == 1 { + inputs.get_mut("msg_1").unwrap().error = Some("refused".into()); + } + let done = inputs.complete(&format!("msg_{index}")).unwrap(); + assert_eq!(done.id, Some(index.to_string())); + assert!(!done.text.is_empty()); + assert!(done.structured.is_some()); + assert!(inputs.pending.is_empty()); + } + assert_eq!(inputs.total(), 4096); + assert_eq!(inputs.failure_code(), Some(EXIT_ERROR)); + assert!(inputs.unfinished().is_empty()); + } + + #[test] + fn capacity_refuses_before_counting_and_retirement_preserves_other_identities() { + let mut inputs = Inputs::default(); + for index in 0..MAX_PENDING { + inputs.admit(input(index)).unwrap(); + } + assert!(inputs.admit(input(100)).is_err()); + assert_eq!(inputs.total(), MAX_PENDING as u64); + inputs.complete("msg_0").unwrap(); + assert!(inputs.contains("msg_31")); + assert!(!inputs.contains("msg_0")); + inputs.admit(input(100)).unwrap(); + inputs.get_mut("msg_31").unwrap().error = Some("existing error".into()); + let remaining = inputs.unfinished(); + assert_eq!(remaining.len(), MAX_PENDING); + assert_eq!(remaining[0].message_id, "msg_1"); + assert_eq!(remaining.last().unwrap().message_id, "msg_100"); + assert_eq!(remaining[30].error.as_deref(), Some("existing error")); + assert!(remaining.iter().all(|input| input.error.is_some())); + assert_eq!(inputs.total(), MAX_PENDING as u64 + 1); + assert!(inputs.is_empty()); + assert_eq!(inputs.failure_code(), Some(EXIT_ERROR)); + } +} diff --git a/crates/cyber-cli/src/commands/exec/mod.rs b/crates/cyber-cli/src/commands/exec/mod.rs index 76569c460..2cea62626 100644 --- a/crates/cyber-cli/src/commands/exec/mod.rs +++ b/crates/cyber-cli/src/commands/exec/mod.rs @@ -5,6 +5,7 @@ mod delegation; mod driver; mod frames; pub(crate) mod goals; +mod inputs; mod last_message; pub(crate) mod report; mod review_run; diff --git a/crates/cyber-cli/src/commands/exec/report.rs b/crates/cyber-cli/src/commands/exec/report.rs index 879b50aac..fc56b8f07 100644 --- a/crates/cyber-cli/src/commands/exec/report.rs +++ b/crates/cyber-cli/src/commands/exec/report.rs @@ -48,6 +48,15 @@ pub struct Run { } impl Run { + /// JSONL already emitted these payloads. Keep billing and aggregate status, + /// while releasing the previous answer's text, tool inputs and denials. + pub(super) fn retire_driver_answer(&mut self) { + self.text = String::new(); + self.structured = None; + self.calls = HashMap::new(); + self.denials = Vec::new(); + } + pub fn new(args: &ExecArgs, started: Instant) -> Self { Self { started, @@ -1173,6 +1182,32 @@ mod tests { ); } + #[test] + fn retiring_a_driver_answer_releases_payloads_without_losing_run_totals_or_error() { + let args = Wrapper::parse_from(["exec", "hi"]).args; + let mut run = Run::new(&args, Instant::now()); + run.input_tokens = 12; + run.output_tokens = 3; + run.cost = 0.5; + run.turns = 2; + run.error = Some("earlier failure".into()); + run.text = "completed answer".repeat(1024); + run.structured = Some(json!({"answer":"completed"})); + run.calls.insert( + "call_1".into(), + ("bash".into(), json!({"command":"exit 1"})), + ); + run.denials.push(json!({"message":"denied"})); + run.retire_driver_answer(); + assert!(run.text.is_empty()); + assert!(run.structured.is_none()); + assert!(run.calls.is_empty()); + assert!(run.denials.is_empty()); + assert_eq!((run.input_tokens, run.output_tokens, run.turns), (12, 3, 2)); + assert_eq!(run.cost, 0.5); + assert_eq!(run.error.as_deref(), Some("earlier failure")); + } + /// A discarded attempt is reported as a `retry` line; only recorded content becomes /// `assistant_text`, so the run's result is the replayed answer alone. #[test] diff --git a/crates/cyber-cli/tests/exec_stream_input.rs b/crates/cyber-cli/tests/exec_stream_input.rs index 991e98448..5cc7a7b9f 100644 --- a/crates/cyber-cli/tests/exec_stream_input.rs +++ b/crates/cyber-cli/tests/exec_stream_input.rs @@ -247,6 +247,34 @@ fn unavailable_snapshots_do_not_fail_a_successful_jsonl_input() { assert_eq!(run.exit(), Some(0)); } +#[test] +fn completed_inputs_release_slots_and_keep_run_accounting_and_correlation() { + const COUNT: usize = 65; + let provider = Provider::start(&["text:completed"; COUNT]); + let mut run = Driven::start(&provider, &[]); + let mut seen = Vec::new(); + let mut messages = std::collections::BTreeSet::new(); + for index in 0..COUNT { + let id = format!("input-{index}"); + run.send(&json!({"type":"user","text":"next","id":id}).to_string()); + let input = run.until("input", &mut seen); + assert_eq!(input["id"], id); + assert_eq!(input["status"], "admitted"); + let result = run.until("result", &mut seen); + assert_eq!(result["id"], id); + assert_eq!(result["message_id"], input["message_id"]); + assert!(messages.insert(result["message_id"].as_str().unwrap().to_owned())); + assert_eq!(result["result"], "completed"); + assert_eq!(result["exit_code"], 0); + seen.clear(); + } + run.send(r#"{"type":"end"}"#); + let end = run.until("end", &mut seen); + assert_eq!(end["inputs"], COUNT); + assert_eq!(end["exit_code"], 0); + assert_eq!(run.exit(), Some(0)); +} + #[test] fn last_message_publication_failure_fails_the_run_without_losing_the_answer() { let provider = Provider::start(&["text:completed answer"]); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 98fea2d47..07aefc522 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -45,12 +45,17 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. the Session without binding to the driver's admitted messages. Track exact input/request authority and test two drivers/owner clients sharing a Session, including unrelated work present before driver startup. -3. **Major: long runs retain completed payloads and output blocks the loop.** - `inputs` retains every completed text/structured result; the pending-input cap - does not bound that history. `Out::line` writes/flushes stdout synchronously - on the same loop that handles controls and lease renewal. Retire emitted - payloads into bounded accounting and use a bounded output owner with explicit - I/O-failure settlement; test long runs, slow readers and broken pipes. +3. **Partly fixed locally: completed history and blocking output.** + Inputs now occupy at most 32 pending message-ID slots; emitted text/structured + results retire into bounded counters and first-failure accounting. Admission + ordinals preserve pending result order on premature termination. The reporter + releases completed answer/tool/denial payloads, and background tracking retains + one exact fork-handback bit rather than every observed admission. Thirty-one + exec unit tests, 17 delegation tests, five forked-command tests and nine JSONL + tests pass, including 65 sequential prompts. `Out::line` still writes/flushes + stdout synchronously on the control/lease loop. Complete bounded output + ownership with explicit I/O-failure settlement, bounded background Job lookup, + slow-reader/broken-pipe acceptance and exact shared-Session ownership. 4. **Fixed locally: last-message publication used a predictable temporary path.** Publication now uses an exclusively created sibling file, synchronizes it and atomically replaces the destination, with automatic temporary-file cleanup on diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md index ba970694e..a8a631f59 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md @@ -16,3 +16,9 @@ - **THEN** it SHALL stage bytes in an exclusively created sibling file and atomically replace the destination without following a pre-existing temporary or destination link - **AND** concurrent writers SHALL publish complete answers and clean up their own temporary files, including after publication failure - **AND** a publication failure SHALL preserve per-input results, report the error in `end`, and fail an otherwise successful aggregate run without replacing an existing interruption or protocol exit code + +#### Scenario: Completed inputs do not consume pending capacity +- **WHEN** a driver emits terminal results across a run longer than its pending-input limit +- **THEN** it SHALL release completed text, structured results and correlation records, retaining only bounded aggregate accounting +- **AND** the limit SHALL apply before each HTTP admission to pending inputs, completion SHALL preserve the exact identities of other pending inputs, and `end.inputs` SHALL still count retired attempts +- **AND** retiring a failed input SHALL NOT erase its aggregate failure; premature termination SHALL emit a terminal result for every remaining pending input diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 04a12bcb9..bfa294302 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -736,3 +736,26 @@ The native workflow now selects these regressions and builds the Linux sandbox helper after installing bubblewrap. External results for these edits are pending. Task 6.3 remains open for hook events and bounded output ownership/backpressure; 6.2/6.4 and all parent acceptance obligations remain open. + +Pending-input ledger continuation (2026-11-09): the JSONL driver now reserves an +exact message-ID slot before HTTP admission, retains at most 32 pending inputs, +and releases each terminal payload after emission. Promoted answers use stable +message IDs instead of indices into completed history. Refused admissions retire +their reserved slot; bounded aggregate counters retain input totals and failure +after payload retirement. Premature termination emits every pending result even +when an input already has an error. Two unit tests pass, including 4,096 payload +retirements and capacity/reused-slot identity checks. All nine real JSONL tests +pass, including 65 sequential correlated prompts and exact final input count. +Admission ordinals preserve pending terminal result order and the first failure +independently of completion order. The reporter also releases emitted completed +answer/tool/denial payloads while preserving billing and aggregate status. Owned +background tracking retains only its exact fork's handback-admitted bit rather +than every observed prompt ID; unrelated/foreign Session admissions cannot set +that bit. Thirty-one exec unit, 17 real delegation, five forked-command and nine +JSONL tests pass. CLI all-target Clippy passes with warnings denied; strict +OpenSpec passes 100 items and lint reports zero errors/25 warnings. Focused review +found the pending-output ordering regression and corrected it before acceptance +of this change. Request/interrupt ownership, bounded background Job lookup, +bounded output/backpressure, hook events and remaining 6.4 acceptance still remain +required. Tasks 6.2–6.4 stay open. Native CI run 42693044871 is on the earlier +9ac06c40 checkpoint; it is not native acceptance of this ledger extension. From c971ac79fcc8c7278ac70c9c18fd9677f5ae3ba2 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 16:47:04 -0400 Subject: [PATCH 55/87] Settle open structured results before terminal drain exit --- crates/cyber-server/src/runtime/drain.rs | 45 +++-- .../src/runtime/structured_turn.rs | 37 ++++ crates/cyber-server/tests/structured_root.rs | 186 ++++++++++++++++++ docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 1 + .../specs/session-runtime/spec.md | 10 + .../complete-daily-coding-surfaces/tasks.md | 30 ++- 7 files changed, 295 insertions(+), 16 deletions(-) diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 5bb6d561d..964651d02 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -71,20 +71,7 @@ pub(crate) async fn run(inner: Arc, id: String, forced: bool, cancel: Can inner.member_status_changed(&id).await; let mut forced = forced; let ended = loop { - // A defect must not leave the Session registered as running forever. Boxed: the - // pass state would otherwise be copied through this frame, which nested - // foreground subagents stack up. - let outcome = std::panic::AssertUnwindSafe(Box::pin(pass(&inner, &id, forced, &cancel))) - .catch_unwind() - .await; - let failure = match outcome { - Ok(Ok(())) => None, - Ok(Err(e)) => Some((error_kind(&e).to_string(), e.to_string())), - Err(_) => Some(( - "internal".to_string(), - "the Drain stopped on an internal error; see the log".to_string(), - )), - }; + let failure = run_pass(&inner, &id, forced, &cancel).await; let succeeded = failure.is_none(); // What idle notices report about this pass, read while its run still counts. let ended = ( @@ -116,6 +103,36 @@ pub(crate) async fn run(inner: Arc, id: String, forced: bool, cancel: Can went_idle(super::Runtime { inner }, &id, ended).await; } +/// Classify pass failures and finish root-result metadata before releasing the +/// current Drain owner or publishing idle. Shutdown keeps resumable attempts open. +async fn run_pass( + inner: &Arc, + id: &str, + forced: bool, + cancel: &CancellationToken, +) -> Option<(String, String)> { + // A defect must not leave the Session registered as running forever. Boxed: + // nested foreground subagents otherwise stack copies of this pass state. + let outcome = std::panic::AssertUnwindSafe(Box::pin(pass(inner, id, forced, cancel))) + .catch_unwind() + .await; + let mut failure = match outcome { + Ok(Ok(())) => None, + Ok(Err(error)) => Some((error_kind(&error).to_string(), error.to_string())), + Err(_) => Some(( + "internal".into(), + "the Drain stopped on an internal error; see the log".into(), + )), + }; + if let Err(error) = inner + .root_result_at_exit(id, failure.as_ref(), cancel.is_cancelled()) + .await + { + failure = Some(("storage".into(), error.to_string())); + } + failure +} + /// The Session's Drain ended with `(hops, status)` for its idle notices. async fn went_idle(runtime: super::Runtime, id: &str, (hops, status): (u32, Option)) { runtime.inner.release_attachment_cache(id).await; diff --git a/crates/cyber-server/src/runtime/structured_turn.rs b/crates/cyber-server/src/runtime/structured_turn.rs index 930672e7e..ec7965747 100644 --- a/crates/cyber-server/src/runtime/structured_turn.rs +++ b/crates/cyber-server/src/runtime/structured_turn.rs @@ -13,6 +13,43 @@ use super::{Handle, Inner, RuntimeError, StructuredOutcome}; const KIND: &str = "structured_output"; impl Inner { + /// A terminal Drain exit must not strand its root input's result. Keep this + /// before releasing the Drain entry, so another pass cannot replace ownership. + /// Shutdown preserves an open attempt for recovery in the next process. + pub(crate) async fn root_result_at_exit( + &self, + session_id: &str, + failure: Option<&(String, String)>, + interrupted: bool, + ) -> Result<(), RuntimeError> { + if self.closed.is_cancelled() { + return Ok(()); + } + let handle = self.handle(session_id).await?; + let mut state = handle.state.lock().await; + if self.closed.is_cancelled() || !state.result.root_open() { + return Ok(()); + } + let result = &state.result; + let value = result.returned.as_ref().map(|(_, value)| value.clone()); + let error = value.is_none().then(|| { + let why = match failure { + Some((kind, message)) => format!("{kind}: {message}"), + None if interrupted => "interrupted".into(), + None => "execution ended before structured_output".into(), + }; + format!("StructuredOutputError: {why}") + }); + let outcome = StructuredOutcome { + message_id: result.input.clone().expect("open root input"), + value, + error, + attempts: result.attempts, + }; + self.commit_locked(&mut state, vec![event(SETTLED, &outcome)])?; + Ok(()) + } + /// After a Turn's calls settle: the input's valid result, or its last failure, ends /// the answer; otherwise the model goes on. pub(crate) async fn root_result_end( diff --git a/crates/cyber-server/tests/structured_root.rs b/crates/cyber-server/tests/structured_root.rs index e637f9a6e..4eddf162e 100644 --- a/crates/cyber-server/tests/structured_root.rs +++ b/crates/cyber-server/tests/structured_root.rs @@ -44,6 +44,192 @@ fn harness(main: Vec>) -> Harness { }) } +#[tokio::test] +async fn interrupted_open_result_is_terminal_and_remains_bound_to_its_input_after_restart() { + let h = harness(vec![ + tools(&[("work", "read", "{}")]), + text("later plain answer"), + ]); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::UntilCancelled); + let id = h.session().await; + let started = h.tools.started.notified(); + let message = ask(&h, &id, "audit", Some(schema())).await; + tokio::time::timeout(std::time::Duration::from_secs(10), started) + .await + .unwrap(); + h.runtime.interrupt(&id).await.unwrap(); + h.settle(&id).await; + let outcome = result_of(&h, &id, &message) + .await + .expect("interrupted input must settle"); + assert_eq!(outcome.value, None); + assert_eq!( + outcome.attempts, 0, + "ordinary tools are not result submissions" + ); + assert!(outcome.error.as_deref().unwrap().contains("interrupted")); + assert_terminal_result_survives_restart(&h, &id, &message, &outcome).await; +} + +#[tokio::test] +async fn a_budget_ending_an_open_correction_records_a_terminal_input_result() { + let h = harness(vec![ + structured("invalid", r#"{"wrong":1}"#), + text("must not dispatch"), + ]); + let id = h + .runtime + .create_session(cyber_server::runtime::CreateSession { + directory: h.repo.display().to_string(), + model: "test/main".into(), + budget: Some(cyber_core::budget::Budget { + max_turns: Some(1), + ..Default::default() + }), + ..Default::default() + }) + .await + .unwrap() + .id; + let message = ask(&h, &id, "audit", Some(schema())).await; + h.settle(&id).await; + let outcome = result_of(&h, &id, &message) + .await + .expect("budget-ended input must settle"); + assert_eq!(outcome.value, None); + assert_eq!(outcome.attempts, 1); + assert!(outcome.error.as_deref().unwrap().contains("budget")); + assert_terminal_result_survives_restart(&h, &id, &message, &outcome).await; +} + +async fn assert_terminal_result_survives_restart( + h: &Harness, + id: &str, + message: &str, + outcome: &StructuredOutcome, +) { + let events = h.store.read_events(id, -1, 200).unwrap().events; + assert_eq!( + events + .iter() + .filter(|event| event.kind == "session.structured_output.settled.1" + && event.data["message_id"] == message) + .count(), + 1 + ); + assert_eq!(h.models.requests("test/main").len(), 1); + let restarted = h.restart(); + let state = restarted.state(id).await.unwrap(); + let recorded = state + .inbox + .iter() + .find(|row| row.message_id == message) + .unwrap(); + assert_eq!(recorded.structured.as_ref(), Some(outcome)); + assert_eq!( + h.models.requests("test/main").len(), + 1, + "lookup never redispatches" + ); +} + +#[tokio::test] +async fn shutdown_keeps_the_open_result_for_explicit_restart_resume() { + let h = harness(vec![ + tools(&[("work", "read", "{}")]), + structured("final", r#"{"summary":"resumed"}"#), + ]); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::UntilCancelled); + let id = h.session().await; + let started = h.tools.started.notified(); + let message = ask(&h, &id, "audit", Some(schema())).await; + tokio::time::timeout(std::time::Duration::from_secs(10), started) + .await + .unwrap(); + h.runtime.shutdown().await; + h.settle(&id).await; + assert!(result_of(&h, &id, &message).await.is_none()); + let restarted = h.restart(); + restarted.resume(&id).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), restarted.wait_idle(&id)) + .await + .unwrap(); + let state = restarted.state(&id).await.unwrap(); + let outcome = state + .inbox + .iter() + .find(|row| row.message_id == message) + .unwrap() + .structured + .clone() + .unwrap(); + assert_eq!(outcome.value, Some(json!({"summary":"resumed"}))); + assert_eq!(outcome.attempts, 1); + assert_eq!(h.models.requests("test/main").len(), 2); +} + +#[tokio::test] +async fn compaction_during_a_correction_keeps_the_input_schema_and_attempt_count() { + let h = Harness::new(Setup { + scripts: vec![ + ( + "test/main", + vec![ + text("earlier answer"), + structured("invalid", r#"{"wrong":1}"#), + error(cyber_llm::ErrorKind::ContextOverflow, "prompt too long"), + structured("final", r#"{"summary":"after compaction"}"#), + ], + ), + ("test/summary", vec![text("compressed earlier work")]), + ], + compaction: cyber_server::runtime::CompactionConfig { + auto: true, + buffer: 100, + keep_tokens: 10_000, + keep_turns: Some(1), + model: None, + }, + ..Setup::default() + }); + let id = h.session().await; + ask(&h, &id, "before", None).await; + h.settle(&id).await; + let message = ask(&h, &id, "audit", Some(schema())).await; + h.settle(&id).await; + let outcome = result_of(&h, &id, &message).await.unwrap(); + assert_eq!(outcome.value, Some(json!({"summary":"after compaction"}))); + assert_eq!(outcome.attempts, 2); + assert!(h.state(&id).await.compacted.is_some()); + assert_eq!(h.models.requests("test/summary").len(), 1); + let requests = h.models.requests("test/main"); + assert_eq!(requests.len(), 4); + let retried = &requests[3]; + assert_eq!( + retried + .tools + .iter() + .find(|tool| tool.name == "structured_output") + .unwrap() + .input_schema, + schema() + ); + assert!( + retried + .system + .concat() + .contains("calling structured_output") + ); +} + #[tokio::test] async fn a_valid_submission_is_the_inputs_result_and_ends_the_answer() { let h = harness(vec![structured("c1", r#"{"summary":"all good"}"#)]); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 07aefc522..82d979e66 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -22,7 +22,7 @@ section 5/6 implementation state; reconcile it at delivery. | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill acceptance passes locally, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | | #93 rules/context, tasks 4.1–4.4 | Implement trusted path-rule discovery/precedence/bounds and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | -| #93 structured results, tasks 5.1–5.4 | #116 merged admission/schema bounds, finalization, retries, durable per-input outcomes and CLI/API/SDK rendering. Three real CLI and five server tests pass after integration. Compaction during an open attempt and budget/interrupt terminal outcomes still need acceptance; do not infer them from restart between completed prompts. | +| #93 structured results, tasks 5.1–5.4 | #116 merged admission/schema bounds, finalization, retries, per-input outcomes and CLI/API/SDK rendering. Runtime acceptance now proves compaction during correction, terminal interruption/budget outcomes and shutdown/resume of an open result. Finish real CLI budget/SIGINT exit precedence and finalizer/terminal-settlement writer/restart cuts without redispatch; lookup after completed prompts does not prove those cuts. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | | #92 protocol/callbacks/producers, tasks 3–6/S/D | Finish contribution ownership, hooks/providers/catalog transforms and TypeScript kit, durable scoped callbacks (`session/prompt` still explicitly refuses), Setup and DirectoryAdded plus combined producer acceptance. Preserve accepted FileChanged and CwdChanged fixes. Complete S1 epoch/directory-key/legacy-deletion and D1 directory-admission/Safe Boundary/child-policy review before their dependent code. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 10d31f77e..cdc5fb108 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -634,6 +634,7 @@ cyber exec --output-schema findings.schema.json --format json "audit the depende - **The schema** must be an object schema (`"type": "object"`), at most 64 KiB and 64 levels deep, with local `$ref`s only (`#/...`). A file that is not JSON or breaks a limit exits 2 before any Session exists; so does combining it with `--command`, `--workflow` or `--goal`. - **How the model answers:** the prompt gets a `structured_output` tool whose input is the schema, and each request reminds the model to end with it. A valid call is the result, and the answer ends there. - **Corrections:** an invalid call returns the validation errors to the model, and an answer that ends without the call gets a reminder. Three failed submissions in all (the first plus two corrections) end the prompt with `StructuredOutputError: ` and exit code 1; there is never a fourth. +- **Interrupted attempts:** once a schema-governed input is promoted, an interruption or budget stop records its terminal error and current submission count before the Drain becomes idle. Lookup after restart reads that outcome without dispatching another request. Shutdown preserves an unfinished attempt for explicit resume; compaction preserves its schema and retry count. - **Output:** text mode prints the object as pretty JSON; `--format json` and the `stream-json` `result` line put it in `result` (`null` on failure, with the error in `error`). Interruption (130), budgets and timeouts (4) keep their usual exit codes. - **API:** `POST /api/v1/sessions/{id}/prompt` takes `"format": {"type": "json_schema", "schema": {…}}` on a root Session's prompt (a subagent has its own result schema). The schema belongs to that prompt only: queued prompts keep their own schemas, results and counts, also across a restart. The outcome is recorded as `session.structured_output.settled.1` (`message_id`, `value` or `error`, `attempts`) and kept on the input's inbox row as `structured`, so a client that lost the reply can read it again. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md index e3bfe7ef1..162382fcd 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md @@ -14,3 +14,13 @@ #### Scenario: Restart between queued schemas - **WHEN** a Session restarts after completing one schema-governed prompt and before promoting another with a different schema - **THEN** each retained input has its own schema, retry count and result and the second cannot reuse the first's successful value + +#### Scenario: Interruption or budget stops an open result +- **WHEN** execution is interrupted or an ordinary budget stops a root prompt before valid finalization +- **THEN** the Drain SHALL durably record the matching input's terminal error and current submission count before releasing its owner or publishing idle +- **AND** an already durable valid finalizer SHALL remain its successful value, result-settlement failure SHALL remain an execution failure, and lookup after restart SHALL NOT redispatch work +- **AND** shutdown SHALL preserve an open attempt for explicit restart recovery rather than inventing terminal cancellation + +#### Scenario: Compaction between validation submissions +- **WHEN** automatic compaction recovers an overflow during a root result correction +- **THEN** the retried request SHALL retain the matching input schema and finalization instruction, without resetting the submitted-attempt count or counting overflow as a submission diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index bfa294302..e1d0f10fa 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -137,7 +137,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - [x] 5.1 Add validated per-prompt root format admission, bounded schema compilation and durable schema/attempt ownership without changing existing child contracts. - [x] 5.2 Add `structured_output` finalization, two correction retries, budget/interrupt handling and durable successful/error result settlement. - [x] 5.3 Add `--output-schema`, API/SDK result fields and text/JSON/stream rendering; preserve unstructured commands and validate option combinations before admission. -- [ ] 5.4 Test valid/invalid schemas, malformed results, retry exhaustion, missing finalizer, compaction, restart/retry identity, budget exits and independent queued prompts with a real server/scripted provider. Done: schema bounds and refusal before admission, malformed and invalid submissions, exhaustion without a fourth request, missing finalizer, a restart between independent queued prompts, retry identity with another schema, and the real binary (`cyber-cli/tests/exec_output_schema.rs`). Open: compaction during an attempt and budget exits. +- [ ] 5.4 Test valid/invalid schemas, malformed results, retry exhaustion, missing finalizer, compaction, restart/retry identity, budget exits and independent queued prompts with a real server/scripted provider. Done: schema bounds and refusal before admission, malformed and invalid submissions, exhaustion without a fourth request, missing finalizer, a restart between independent queued prompts, retry identity with another schema, the real binary (`cyber-cli/tests/exec_output_schema.rs`), runtime compaction during correction, durable interruption/budget outcomes, and shutdown/resume of an open result. Open: real CLI budget/SIGINT exit precedence and finalizer/terminal-settlement writer/restart cuts without redispatch. ## 6. Streaming JSONL input @@ -759,3 +759,31 @@ of this change. Request/interrupt ownership, bounded background Job lookup, bounded output/backpressure, hook events and remaining 6.4 acceptance still remain required. Tasks 6.2–6.4 stay open. Native CI run 42693044871 is on the earlier 9ac06c40 checkpoint; it is not native acceptance of this ledger extension. + +Structured terminal-result continuation (2026-11-09): new runtime regressions +first reproduced absent input outcomes after interruption and budget exhaustion +during correction (five existing tests passed, two regressions failed). Drain +exit now records the matching open root input's outcome before releasing its +entry or publishing idle. It preserves a durable valid finalizer, records the +current submission count on error, and leaves shutdown attempts open for explicit +restart resume. Metadata settlement failures remain execution failures. Pass +execution/error classification is extracted to keep the existing cognitive +complexity limit. Four added cases prove interrupted/budget outcomes and lookup +after restart, schema/attempt preservation through actual overflow compaction, +and shutdown followed by explicit resume without resetting the result attempt. + +Final local checks: 94 runtime regressions pass (13 budget, 19 context/compaction, +53 runtime and nine structured-root); three structured-output CLI and nine JSONL +tests pass. Server all-target Clippy passes with warnings denied, as do format, +whitespace and strict OpenSpec (100 items; lint zero errors/25 warnings). An +earlier Clippy run ran out of disk and the broader test command could not create +its log; neither is a passing attempt. Cargo package cleanup removed 23.2 GiB of +rebuildable server artifacts, and the checks above passed after cleanup with +incremental compilation disabled. Finalizer/terminal-outcome writer/restart cuts +and real CLI budget/SIGINT precedence remain open under 5.4. No new public receipt +or API/SDK field was added by this correction. + +Native run 42693044871 passed all selected native, managed Git App/CLI and JSONL +steps on Linux x64 and ARM for 9ac06c40. macOS Intel/ARM jobs remain queued at this +audit; they have no passing result. The ledger and structured terminal-result +extensions need their own current-source native acceptance before delivery. From 03b4526ac3aeb4b9bd4ff1b58393c297026bc6a9 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 18:55:31 -0400 Subject: [PATCH 56/87] Reconcile durable structured results before resuming inference --- .github/workflows/non-git-snapshot-proof.yml | 13 +- crates/cyber-cli/tests/exec_output_schema.rs | 123 ++++++++++- crates/cyber-server/src/runtime/drain.rs | 6 + crates/cyber-server/tests/structured_root.rs | 199 +++++++++++++++++- docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 1 + .../specs/session-runtime/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 32 ++- 8 files changed, 370 insertions(+), 12 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index f6403240f..f52ec4559 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -8,12 +8,17 @@ on: - crates/cyber-app/tests/support/scripted_model.rs - crates/cyber-server/src/runtime/snapshot*.rs - crates/cyber-server/src/runtime/rewind*.rs + - crates/cyber-server/src/runtime/drain.rs + - crates/cyber-server/src/runtime/structured*.rs + - crates/cyber-server/src/runtime/model.rs + - crates/cyber-server/tests/structured_root.rs - crates/cyber-server/src/http/sessions.rs - crates/cyber-app/tests/snapshot_git_runtime.rs - crates/cyber-cli/Cargo.toml - crates/cyber-cli/src/** - crates/cyber-cli/tests/branches_cli.rs - crates/cyber-cli/tests/exec_stream_input.rs + - crates/cyber-cli/tests/exec_output_schema.rs - crates/cyber-cli/tests/support/** - crates/cyber-server/src/workspace.rs - crates/cyber-core/src/binary/native/** @@ -78,8 +83,13 @@ jobs: shell: bash run: | set -euo pipefail - cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::exec::last_message::tests -- --nocapture 2>&1 | tee jsonl-driver-proof.log + cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::exec:: -- --nocapture 2>&1 | tee jsonl-driver-proof.log cargo test --locked -p cyber-cli --features native-git-acceptance --test exec_stream_input -- --nocapture 2>&1 | tee -a jsonl-driver-proof.log + - name: Root structured result recovery and CLI stop regressions + shell: bash + run: | + set -euo pipefail + cargo test --locked -p cyber-server -p cyber-cli --features cyber-cli/native-git-acceptance --test structured_root --test exec_output_schema -- --nocapture 2>&1 | tee structured-result-proof.log - name: Retain native evidence if: always() uses: actions/upload-artifact@v4 @@ -92,4 +102,5 @@ jobs: git-runtime-proof.log git-cli-recovery-proof.log jsonl-driver-proof.log + structured-result-proof.log if-no-files-found: warn diff --git a/crates/cyber-cli/tests/exec_output_schema.rs b/crates/cyber-cli/tests/exec_output_schema.rs index 89b110a09..c16a57d64 100644 --- a/crates/cyber-cli/tests/exec_output_schema.rs +++ b/crates/cyber-cli/tests/exec_output_schema.rs @@ -7,7 +7,7 @@ use std::collections::VecDeque; use std::io::{Read, Write}; use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; -use std::process::{Command, Output, Stdio}; +use std::process::{Child, Command, Output, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; @@ -37,7 +37,7 @@ impl Provider { let server = thread::spawn(move || { while !stop.load(Ordering::SeqCst) { match listener.accept() { - Ok((stream, _)) => answer(stream, &seen, &script), + Ok((stream, _)) => answer(stream, &seen, &script, &stop), Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { thread::sleep(Duration::from_millis(5)); } @@ -67,7 +67,12 @@ impl Drop for Provider { } } -fn answer(mut stream: TcpStream, turns: &Mutex>, script: &Mutex>) { +fn answer( + mut stream: TcpStream, + turns: &Mutex>, + script: &Mutex>, + done: &AtomicBool, +) { stream.set_nonblocking(false).unwrap(); stream .set_read_timeout(Some(Duration::from_secs(5))) @@ -78,6 +83,12 @@ fn answer(mut stream: TcpStream, turns: &Mutex>, script: &Mutex { json!({"choices":[{"index":0,"delta":{"content":text},"finish_reason":"stop"}]}) @@ -139,10 +150,11 @@ impl Env { Self { _dir: dir, root } } - fn exec(&self, flags: &[&str]) -> Output { + fn command(&self, flags: &[&str]) -> Command { let helper = PathBuf::from(env!("CARGO_BIN_EXE_cyber")).with_file_name("cyber-sandbox-exec"); - Command::new(env!("CARGO_BIN_EXE_cyber")) + let mut command = Command::new(env!("CARGO_BIN_EXE_cyber")); + command .args(["--cwd", self.root.join("repo").to_str().unwrap()]) .args(["exec", "--embedded", "--model", "corp/main"]) .args(flags) @@ -154,9 +166,12 @@ impl Env { .env("CYBER_SANDBOX_HELPER", helper) .env("PATH", std::env::var_os("PATH").unwrap_or_default()) .current_dir(&self.root) - .stdin(Stdio::null()) - .output() - .unwrap() + .stdin(Stdio::null()); + command + } + + fn exec(&self, flags: &[&str]) -> Output { + self.command(flags).output().unwrap() } fn schema(&self) -> String { @@ -174,6 +189,98 @@ fn stdout(output: &Output) -> String { String::from_utf8_lossy(&output.stdout).into_owned() } +fn collect(mut child: Child) -> Output { + for _ in 0..200 { + if child.try_wait().unwrap().is_some() { + return child.wait_with_output().unwrap(); + } + thread::sleep(Duration::from_millis(100)); + } + let _ = child.kill(); + let output = child.wait_with_output().unwrap(); + panic!("structured exec did not settle after cancellation: {output:?}"); +} + +#[cfg(unix)] +fn wait_for_turns(provider: &Provider, child: &mut Child, count: usize) { + for _ in 0..2000 { + if provider.turns() >= count { + return; + } + assert!( + child.try_wait().unwrap().is_none(), + "exec exited before the requested correction" + ); + thread::sleep(Duration::from_millis(5)); + } + let _ = child.kill(); + let _ = child.wait(); + panic!("the provider did not receive the correction"); +} + +#[test] +fn budgets_and_timeouts_keep_exit_4_when_the_open_structured_attempt_is_interrupted() { + for (stopping, reason) in [ + (["--max-tokens", "1"], "budget_exceeded"), + (["--timeout", "2s"], "timeout"), + ] { + let provider = Provider::start(&[r#"{"wrong":1}"#, "wait"]); + let env = Env::new(&provider); + let schema = env.schema(); + let mut flags = vec!["--output-schema", &schema, "--format", "json"]; + flags.extend(stopping); + flags.push("audit"); + let output = collect( + env.command(&flags) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(), + ); + assert_eq!(output.status.code(), Some(4), "{output:?}"); + let result: Value = serde_json::from_str(stdout(&output).trim()).unwrap(); + assert_eq!(result["exit_code"], 4, "{result}"); + assert_eq!(result["stop_reason"], reason, "{result}"); + assert_eq!(result["result"], Value::Null); + assert!(result["error"].as_str().is_some(), "{result}"); + assert!((1..=2).contains(&provider.turns())); + } +} + +#[cfg(unix)] +#[test] +fn sigint_during_a_structured_correction_keeps_exit_130() { + let provider = Provider::start(&[r#"{"wrong":1}"#, "wait"]); + let env = Env::new(&provider); + let mut child = env + .command(&[ + "--output-schema", + &env.schema(), + "--format", + "json", + "audit", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + wait_for_turns(&provider, &mut child, 2); + // This is the owned test child, after its correction reached the provider. + let signal = Command::new("kill") + .args(["-INT", &child.id().to_string()]) + .status() + .unwrap(); + assert!(signal.success()); + let output = collect(child); + assert_eq!(output.status.code(), Some(130), "{output:?}"); + let result: Value = serde_json::from_str(stdout(&output).trim()).unwrap(); + assert_eq!(result["exit_code"], 130); + assert_eq!(result["stop_reason"], "interrupted"); + assert_eq!(result["result"], Value::Null); + assert!(result["error"].as_str().is_some()); + assert_eq!(provider.turns(), 2); +} + #[test] fn the_validated_object_is_the_result() { let provider = Provider::start(&[r#"{"summary":"no issues"}"#, r#"{"summary":"again"}"#]); diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 964651d02..c41fd1966 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -328,6 +328,12 @@ async fn start_pass( let depth = super::messaging::hops::restored_depth(&state); inner.raise_mcp_depth(&state.info.id, depth); } + // A finalizer or exhausted correction may be durable while its terminal + // outcome append was interrupted. Reconcile it before any model dispatch; + // queued new input can still run, but recovery does not force another Turn. + if inner.root_result_end(handle).await?.is_some() { + return Ok(Some(false)); + } Ok(match resume_after_wait(inner, handle, cancel).await? { Some(false) => None, Some(true) => Some(true), diff --git a/crates/cyber-server/tests/structured_root.rs b/crates/cyber-server/tests/structured_root.rs index 4eddf162e..d976c88ee 100644 --- a/crates/cyber-server/tests/structured_root.rs +++ b/crates/cyber-server/tests/structured_root.rs @@ -4,7 +4,9 @@ mod support; -use cyber_server::runtime::{Admission, Delivery, RuntimeError, StructuredOutcome}; +use cyber_server::runtime::{ + Admission, Delivery, Runtime, RuntimeError, SessionState, StructuredOutcome, +}; use serde_json::{Value, json}; use support::*; @@ -29,6 +31,10 @@ async fn ask(h: &Harness, id: &str, text: &str, schema: Option) -> String async fn result_of(h: &Harness, id: &str, message: &str) -> Option { let state = h.state(id).await; + outcome_from_state(&state, message) +} + +fn outcome_from_state(state: &SessionState, message: &str) -> Option { let row = state .inbox .iter() @@ -44,6 +50,197 @@ fn harness(main: Vec>) -> Harness { }) } +#[tokio::test] +async fn a_durable_finalizer_reconciles_a_terminal_writer_cut_without_another_request() { + assert_terminal_writer_cut(true).await; +} + +#[tokio::test] +async fn exhausted_submissions_reconcile_a_terminal_writer_cut_without_a_fourth_request() { + assert_terminal_writer_cut(false).await; +} + +fn prevent_terminal_outcome(h: &Harness) { + h.store.transaction(|db| { + db.execute_batch("CREATE TRIGGER reject_structured_terminal BEFORE INSERT ON event WHEN NEW.type='session.structured_output.settled.1' BEGIN SELECT RAISE(ABORT,'fixture before structured terminal'); END")?; + Ok(()) + }).unwrap(); +} + +fn allow_terminal_outcome(h: &Harness) { + h.store + .transaction(|db| { + db.execute_batch("DROP TRIGGER reject_structured_terminal")?; + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn writer_cut_recovery_settles_the_previous_result_before_promoting_a_new_schema() { + let h = harness(vec![ + structured("old", r#"{"summary":"original"}"#), + structured("new", r#"{"count":2}"#), + ]); + prevent_terminal_outcome(&h); + let id = h.session().await; + let first = ask(&h, &id, "first", Some(schema())).await; + h.settle(&id).await; + assert!(result_of(&h, &id, &first).await.is_none()); + assert_eq!(h.models.requests("test/main").len(), 1); + h.runtime.shutdown().await; + allow_terminal_outcome(&h); + let restarted = h.restart(); + let other = + json!({"type":"object","properties":{"count":{"type":"integer"}},"required":["count"]}); + let mut next = Admission::text("second", Delivery::Queue); + next.output_schema = Some(other.clone()); + next.resume = false; + let second = restarted.admit(&id, next).await.unwrap().message_id; + assert_eq!(h.models.requests("test/main").len(), 1); + restarted.resume(&id).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), restarted.wait_idle(&id)) + .await + .unwrap(); + let state = restarted.state(&id).await.unwrap(); + let old = outcome_from_state(&state, &first).unwrap(); + let new = outcome_from_state(&state, &second).unwrap(); + assert_eq!(old.value, Some(json!({"summary":"original"}))); + assert_eq!(new.value, Some(json!({"count":2}))); + assert_eq!((old.attempts, new.attempts), (1, 1)); + assert_eq!((&old.error, &new.error), (&None, &None)); + let requests = h.models.requests("test/main"); + assert_eq!( + requests.len(), + 2, + "only the new prompt dispatches after recovery" + ); + assert_eq!( + requests[1] + .tools + .iter() + .find(|tool| tool.name == "structured_output") + .unwrap() + .input_schema, + other + ); + let events = h.store.read_events(&id, -1, 200).unwrap().events; + let settled = events + .iter() + .find(|event| { + event.kind == "session.structured_output.settled.1" && event.data["message_id"] == first + }) + .unwrap(); + let promoted = events + .iter() + .find(|event| { + event.kind == "session.prompt.promoted.1" && event.data["message_id"] == second + }) + .unwrap(); + assert!(settled.seq < promoted.seq); + restarted.shutdown().await; +} + +async fn prepare_terminal_writer_cut(valid: bool) -> (Harness, String, String, usize) { + let mut script = if valid { + vec![structured("valid", r#"{"summary":"durable value"}"#)] + } else { + vec![ + structured("bad1", r#"{"wrong":1}"#), + structured("bad2", r#"{"summary":3}"#), + structured("bad3", "{}"), + ] + }; + script.push(text("fresh prompt answer")); + let h = harness(script); + prevent_terminal_outcome(&h); + let id = h.session().await; + let message = ask(&h, &id, "audit", Some(schema())).await; + h.settle(&id).await; + let attempts = if valid { 1 } else { 3 }; + assert_eq!(h.models.requests("test/main").len(), attempts); + assert!(result_of(&h, &id, &message).await.is_none()); + let state = h.state(&id).await; + if valid { + assert_eq!( + state.structured_result(), + Some(&json!({"summary":"durable value"})) + ); + } else { + assert!(state.structured_result_error().contains("Invalid result")); + } + h.runtime.shutdown().await; + allow_terminal_outcome(&h); + (h, id, message, attempts) +} + +async fn assert_terminal_writer_cut(valid: bool) { + let (h, id, message, attempts) = prepare_terminal_writer_cut(valid).await; + let restarted = h.restart(); + restarted.resume(&id).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), restarted.wait_idle(&id)) + .await + .unwrap(); + assert_eq!( + h.models.requests("test/main").len(), + attempts, + "metadata recovery must not reopen inference" + ); + let state = restarted.state(&id).await.unwrap(); + let outcome = outcome_from_state(&state, &message).expect("reconciled result"); + assert_reconciled_outcome(&outcome, valid, attempts); + let events = h.store.read_events(&id, -1, 200).unwrap().events; + assert_eq!( + events + .iter() + .filter(|event| event.kind == "session.structured_output.settled.1") + .count(), + 1 + ); + assert_fresh_plain_prompt(&h, &restarted, &id, &message, &outcome, attempts).await; + restarted.shutdown().await; +} + +fn assert_reconciled_outcome(outcome: &StructuredOutcome, valid: bool, attempts: usize) { + assert_eq!(outcome.attempts as usize, attempts); + if valid { + assert_eq!(outcome.value, Some(json!({"summary":"durable value"}))); + assert_eq!(outcome.error, None); + } else { + assert_eq!(outcome.value, None); + assert!(outcome.error.as_deref().unwrap().contains("Invalid result")); + } +} + +async fn assert_fresh_plain_prompt( + h: &Harness, + restarted: &Runtime, + id: &str, + message: &str, + outcome: &StructuredOutcome, + attempts: usize, +) { + restarted + .admit(id, Admission::text("new plain prompt", Delivery::Queue)) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), restarted.wait_idle(id)) + .await + .unwrap(); + assert_eq!(h.models.requests("test/main").len(), attempts + 1); + assert!( + h.models + .requests("test/main") + .last() + .unwrap() + .tools + .iter() + .all(|tool| tool.name != "structured_output") + ); + let state = restarted.state(id).await.unwrap(); + assert_eq!(outcome_from_state(&state, message).as_ref(), Some(outcome)); +} + #[tokio::test] async fn interrupted_open_result_is_terminal_and_remains_bound_to_its_input_after_restart() { let h = harness(vec![ diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 82d979e66..319658470 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -22,7 +22,7 @@ section 5/6 implementation state; reconcile it at delivery. | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill acceptance passes locally, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | | #93 rules/context, tasks 4.1–4.4 | Implement trusted path-rule discovery/precedence/bounds and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | -| #93 structured results, tasks 5.1–5.4 | #116 merged admission/schema bounds, finalization, retries, per-input outcomes and CLI/API/SDK rendering. Runtime acceptance now proves compaction during correction, terminal interruption/budget outcomes and shutdown/resume of an open result. Finish real CLI budget/SIGINT exit precedence and finalizer/terminal-settlement writer/restart cuts without redispatch; lookup after completed prompts does not prove those cuts. | +| #93 structured results, tasks 5.1–5.4 | #116 merged admission/schema bounds, finalization, retries, per-input outcomes and CLI/API/SDK rendering. Runtime acceptance proves compaction, terminal interruption/budget outcomes, shutdown/resume and recovery from a failed terminal append before queued schema promotion. Real CLI budget/timeout exit 4 and Unix SIGINT exit 130 pass locally. Finish post-commit acknowledgement-loss and steering ownership acceptance plus current-source platform checks; lookup after completed prompts alone does not prove those cuts. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | | #92 protocol/callbacks/producers, tasks 3–6/S/D | Finish contribution ownership, hooks/providers/catalog transforms and TypeScript kit, durable scoped callbacks (`session/prompt` still explicitly refuses), Setup and DirectoryAdded plus combined producer acceptance. Preserve accepted FileChanged and CwdChanged fixes. Complete S1 epoch/directory-key/legacy-deletion and D1 directory-admission/Safe Boundary/child-policy review before their dependent code. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index cdc5fb108..2032fbc8f 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -635,6 +635,7 @@ cyber exec --output-schema findings.schema.json --format json "audit the depende - **How the model answers:** the prompt gets a `structured_output` tool whose input is the schema, and each request reminds the model to end with it. A valid call is the result, and the answer ends there. - **Corrections:** an invalid call returns the validation errors to the model, and an answer that ends without the call gets a reminder. Three failed submissions in all (the first plus two corrections) end the prompt with `StructuredOutputError: ` and exit code 1; there is never a fourth. - **Interrupted attempts:** once a schema-governed input is promoted, an interruption or budget stop records its terminal error and current submission count before the Drain becomes idle. Lookup after restart reads that outcome without dispatching another request. Shutdown preserves an unfinished attempt for explicit resume; compaction preserves its schema and retry count. +- **Recovering a terminal write:** if the finalizer or exhausted submissions are durable but their terminal-outcome append failed, resume first settles that unfinished result without another provider request. Any queued new prompt then uses its own schema and attempt count. - **Output:** text mode prints the object as pretty JSON; `--format json` and the `stream-json` `result` line put it in `result` (`null` on failure, with the error in `error`). Interruption (130), budgets and timeouts (4) keep their usual exit codes. - **API:** `POST /api/v1/sessions/{id}/prompt` takes `"format": {"type": "json_schema", "schema": {…}}` on a root Session's prompt (a subagent has its own result schema). The schema belongs to that prompt only: queued prompts keep their own schemas, results and counts, also across a restart. The outcome is recorded as `session.structured_output.settled.1` (`message_id`, `value` or `error`, `attempts`) and kept on the input's inbox row as `structured`, so a client that lost the reply can read it again. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md index 162382fcd..06dc5a914 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md @@ -24,3 +24,9 @@ #### Scenario: Compaction between validation submissions - **WHEN** automatic compaction recovers an overflow during a root result correction - **THEN** the retried request SHALL retain the matching input schema and finalization instruction, without resetting the submitted-attempt count or counting overflow as a submission + +#### Scenario: Durable finalization precedes a failed terminal outcome append +- **GIVEN** a root prompt's valid finalizer or exhausted submissions are durable but its terminal outcome append failed +- **WHEN** an explicit resume recovers that unfinished result after restart +- **THEN** it SHALL settle the original value or exhausted error before model dispatch, preserving its exact input and attempt count without another provider request +- **AND** any new queued prompt SHALL be promoted only after that reconciliation, retaining its own schema and result; ordinary result lookup SHALL never redispatch work diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index e1d0f10fa..57be72597 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -137,7 +137,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - [x] 5.1 Add validated per-prompt root format admission, bounded schema compilation and durable schema/attempt ownership without changing existing child contracts. - [x] 5.2 Add `structured_output` finalization, two correction retries, budget/interrupt handling and durable successful/error result settlement. - [x] 5.3 Add `--output-schema`, API/SDK result fields and text/JSON/stream rendering; preserve unstructured commands and validate option combinations before admission. -- [ ] 5.4 Test valid/invalid schemas, malformed results, retry exhaustion, missing finalizer, compaction, restart/retry identity, budget exits and independent queued prompts with a real server/scripted provider. Done: schema bounds and refusal before admission, malformed and invalid submissions, exhaustion without a fourth request, missing finalizer, a restart between independent queued prompts, retry identity with another schema, the real binary (`cyber-cli/tests/exec_output_schema.rs`), runtime compaction during correction, durable interruption/budget outcomes, and shutdown/resume of an open result. Open: real CLI budget/SIGINT exit precedence and finalizer/terminal-settlement writer/restart cuts without redispatch. +- [ ] 5.4 Test valid/invalid schemas, malformed results, retry exhaustion, missing finalizer, compaction, restart/retry identity, budget exits and independent queued prompts with a real server/scripted provider. Done: schema bounds and refusal before admission, malformed and invalid submissions, exhaustion without a fourth request, missing finalizer, restart between independent queued prompts, retry identity with another schema, runtime compaction during correction, durable interruption/budget outcomes, shutdown/resume of an open result, failed terminal-append recovery before another dispatch/queued schema promotion, and real CLI budget/timeout exit 4 and Unix SIGINT exit 130. Open: post-commit terminal acknowledgement loss, steering ownership and current-source platform acceptance. ## 6. Streaming JSONL input @@ -787,3 +787,33 @@ Native run 42693044871 passed all selected native, managed Git App/CLI and JSONL steps on Linux x64 and ARM for 9ac06c40. macOS Intel/ARM jobs remain queued at this audit; they have no passing result. The ledger and structured terminal-result extensions need their own current-source native acceptance before delivery. + +Structured terminal-write recovery continuation (2026-11-09): a new fault fixture +first reproduced two extra provider requests when a validated finalizer was +durable but its terminal append had failed. Start-of-pass recovery now settles +that valid result or the exhausted correction error before model dispatch; new +queued input remains eligible after reconciliation. Three runtime cases prove +valid/exhausted recovery without another request, preservation of later plain +prompt behavior, and old-result settlement before promoting a queued different +schema. The initial exhausted fixture repeated identical invalid calls and hit +the existing doom-loop approval (confirmed by its persisted permission event), +then timed out. Distinct invalid submissions now reach the intended boundary; +the repeated-tool safeguard is unchanged. + +Real CLI cases now gate a correction at the provider and verify max-token and +wall-time termination retain exit 4, while Unix SIGINT retains 130 despite the +structured terminal error. The direct unsafe signal call was rejected by the +configured unsafe-code lint and replaced with the existing owned-child `kill +-INT` fixture pattern. Fixture helpers are split by preparation, reconciliation +and new-input checks to satisfy the cognitive complexity limit. These are +resolved intermediate failures, not passing attempts. + +Final checks: 111 regressions pass in one server/CLI feature-unified run (97 +runtime cases and 14 real CLI cases); combined server/CLI all-target Clippy passes +with warnings denied, as do formatting, whitespace and strict OpenSpec (100 +items; lint zero errors/25 warnings). The native workflow selects all exec units +plus the structured-root/CLI-stop cases on its four hosts and retains their log. +This edit is not current-source external acceptance: run 42693044871 remains on +9ac06c40 (Linux x64/ARM passed; macOS jobs queued at the preceding audit). No new +API/SDK field or CLI flag was added. SDK/plugin/full-workspace checks and Windows +execution were not rerun here; the SIGINT case is explicitly Unix-only. From 1be03642cbfb23bab4bbc7c5d58b4115c5d8ce37 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 20:25:44 -0400 Subject: [PATCH 57/87] Preserve structured result ownership across steering --- .github/workflows/non-git-snapshot-proof.yml | 5 +- crates/cyber-cli/tests/exec_stream_input.rs | 123 +++++++- crates/cyber-server/src/runtime/drain.rs | 4 + crates/cyber-server/src/runtime/structured.rs | 18 ++ .../src/runtime/structured_turn.rs | 62 ++-- crates/cyber-server/tests/structured_root.rs | 271 ++++++++++++++++++ .../support/structured_terminal_fault.rs | 67 +++++ docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 1 + .../specs/session-runtime/spec.md | 11 + .../complete-daily-coding-surfaces/tasks.md | 34 ++- 11 files changed, 572 insertions(+), 26 deletions(-) create mode 100644 crates/cyber-server/tests/support/structured_terminal_fault.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index f52ec4559..7255a8619 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -12,6 +12,7 @@ on: - crates/cyber-server/src/runtime/structured*.rs - crates/cyber-server/src/runtime/model.rs - crates/cyber-server/tests/structured_root.rs + - crates/cyber-server/tests/support/structured_terminal_fault.rs - crates/cyber-server/src/http/sessions.rs - crates/cyber-app/tests/snapshot_git_runtime.rs - crates/cyber-cli/Cargo.toml @@ -31,7 +32,9 @@ permissions: contents: read concurrency: group: non-git-snapshots-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + # Let explicit native proofs finish and retain evidence while a newer run waits. + # Pull-request checks still cancel obsolete heads. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: native: name: Snapshot native (${{ matrix.os }}) diff --git a/crates/cyber-cli/tests/exec_stream_input.rs b/crates/cyber-cli/tests/exec_stream_input.rs index 5cc7a7b9f..59c18b03e 100644 --- a/crates/cyber-cli/tests/exec_stream_input.rs +++ b/crates/cyber-cli/tests/exec_stream_input.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use std::sync::{Arc, Mutex}; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; use serde_json::{Value, json}; @@ -22,6 +22,13 @@ struct Provider { turns: Arc>>, done: Arc, server: Option>, + gate: Arc, +} + +#[derive(Default)] +struct Gate { + reached: AtomicBool, + released: AtomicBool, } impl Provider { @@ -33,11 +40,13 @@ impl Provider { Arc::new(Mutex::new(script.iter().map(|s| (*s).to_owned()).collect())); let turns = Arc::new(Mutex::new(Vec::new())); let done = Arc::new(AtomicBool::new(false)); + let gate = Arc::new(Gate::default()); + let held = gate.clone(); let (seen, stop) = (turns.clone(), done.clone()); let server = thread::spawn(move || { while !stop.load(Ordering::SeqCst) { match listener.accept() { - Ok((stream, _)) => answer(stream, &seen, &script), + Ok((stream, _)) => answer(stream, &seen, &script, &held, &stop), Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { thread::sleep(Duration::from_millis(5)); } @@ -50,12 +59,28 @@ impl Provider { turns, done, server: Some(server), + gate, } } fn turns(&self) -> Vec { self.turns.lock().unwrap().clone() } + + fn wait_for_gate(&self) { + let deadline = Instant::now() + Duration::from_secs(10); + while !self.gate.reached.load(Ordering::SeqCst) { + assert!( + Instant::now() < deadline, + "provider request did not reach its gate" + ); + thread::sleep(Duration::from_millis(5)); + } + } + + fn release(&self) { + self.gate.released.store(true, Ordering::SeqCst); + } } impl Drop for Provider { @@ -67,7 +92,13 @@ impl Drop for Provider { } } -fn answer(mut stream: TcpStream, turns: &Mutex>, script: &Mutex>) { +fn answer( + mut stream: TcpStream, + turns: &Mutex>, + script: &Mutex>, + gate: &Gate, + done: &AtomicBool, +) { stream.set_nonblocking(false).unwrap(); stream .set_read_timeout(Some(Duration::from_secs(5))) @@ -86,6 +117,16 @@ fn answer(mut stream: TcpStream, turns: &Mutex>, script: &Mutex { + gate.reached.store(true, Ordering::SeqCst); + while !gate.released.load(Ordering::SeqCst) && !done.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(5)); + } + text + } + None => next.as_str(), + }; match next.strip_prefix("text:") { Some(text) => { json!({"choices":[{"index":0,"delta":{"content":text},"finish_reason":"stop"}]}) @@ -230,6 +271,82 @@ impl Driven { } } +#[test] +fn steering_a_structured_input_reports_its_superseded_error_and_the_new_inputs_value() { + let provider = Provider::start(&[ + "gate:structured_output {\"wrong\":1}", + "structured_output {\"summary\":\"latest\"}", + ]); + let mut schema = tempfile::NamedTempFile::new().unwrap(); + schema.write_all(br#"{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}"#).unwrap(); + let path = schema.path().display().to_string(); + let mut run = Driven::start(&provider, &["--output-schema", &path]); + let mut seen = Vec::new(); + run.send(r#"{"type":"user","text":"one","id":"first"}"#); + let first_input = run.until("input", &mut seen); + provider.wait_for_gate(); + run.send(r#"{"type":"user","text":"replace it","id":"next","delivery":"steer"}"#); + let next_input = run.until("input", &mut seen); + assert_eq!( + next_input["status"], "admitted", + "stdin stays live during inference" + ); + run.send(r#"{"type":"end"}"#); + provider.release(); + let first = run.until("result", &mut seen); + assert_eq!(first["id"], "first"); + assert_eq!(first["message_id"], first_input["message_id"]); + assert_eq!(first["result"], Value::Null); + assert_eq!( + first["error"], + format!( + "StructuredOutputError: superseded by input {}", + next_input["message_id"].as_str().unwrap() + ) + ); + assert_eq!(first["exit_code"], 1); + let next = run.until("result", &mut seen); + assert_eq!(next["id"], "next"); + assert_eq!(next["message_id"], next_input["message_id"]); + assert_eq!(next["result"], json!({"summary":"latest"})); + assert_eq!(next["error"], Value::Null); + assert_eq!(next["exit_code"], 0); + let end = run.until("end", &mut seen); + assert_eq!( + (end["inputs"].as_u64(), end["exit_code"].as_u64()), + (Some(2), Some(1)) + ); + assert_eq!(run.exit(), Some(1)); + assert_eq!(provider.turns().len(), 2); +} + +#[test] +fn unstructured_steering_keeps_each_completed_answer_correlated() { + let provider = Provider::start(&["gate:text:first answer", "text:steered answer"]); + let mut run = Driven::start(&provider, &[]); + let mut seen = Vec::new(); + run.send(r#"{"type":"user","text":"one","id":"first"}"#); + run.until("input", &mut seen); + provider.wait_for_gate(); + run.send(r#"{"type":"user","text":"refine","id":"next","delivery":"steer"}"#); + assert_eq!(run.until("input", &mut seen)["status"], "admitted"); + run.send(r#"{"type":"end"}"#); + provider.release(); + let first = run.until("result", &mut seen); + assert_eq!( + (first["id"].as_str(), first["result"].as_str()), + (Some("first"), Some("first answer")) + ); + let next = run.until("result", &mut seen); + assert_eq!( + (next["id"].as_str(), next["result"].as_str()), + (Some("next"), Some("steered answer")) + ); + assert_eq!(run.until("end", &mut seen)["exit_code"], 0); + assert_eq!(run.exit(), Some(0)); + assert_eq!(provider.turns().len(), 2); +} + #[test] fn unavailable_snapshots_do_not_fail_a_successful_jsonl_input() { let provider = Provider::start(&["text:completed without snapshots"]); diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index c41fd1966..9f50c60ff 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -646,7 +646,11 @@ async fn promote( let depth = super::messaging::hops::promoted_depth(&state, &ids); let mut events = Vec::new(); let mut stops = Vec::new(); + let mut outcomes = super::structured_turn::promotion_outcomes(&state, &ids); for message_id in ids { + if let Some(outcome) = outcomes.remove(&message_id) { + events.push(outcome); + } // UserPromptSubmit context and stop request land immediately before their // prompt in the same commit. if let Some(text) = state.hook_contexts.get(&message_id) { diff --git a/crates/cyber-server/src/runtime/structured.rs b/crates/cyber-server/src/runtime/structured.rs index 8372bf27b..8f873f703 100644 --- a/crates/cyber-server/src/runtime/structured.rs +++ b/crates/cyber-server/src/runtime/structured.rs @@ -198,6 +198,24 @@ impl ResultState { self.input.is_some() && !self.settled } + /// Finish only this open root input. A durable valid finalizer wins over + /// interruption or replacement; otherwise retain an explicit terminal error. + pub fn terminal_outcome(&self, reason: &str) -> Option { + if !self.root_open() { + return None; + } + let value = self.returned.as_ref().map(|(_, value)| value.clone()); + let error = value + .is_none() + .then(|| format!("StructuredOutputError: {reason}")); + Some(StructuredOutcome { + message_id: self.input.clone().expect("open root input"), + value, + error, + attempts: self.attempts, + }) + } + /// A child's returned result ends its Drain; a root input's does not stop the next. pub fn blocks_drain(&self) -> bool { self.input.is_none() && self.returned.is_some() diff --git a/crates/cyber-server/src/runtime/structured_turn.rs b/crates/cyber-server/src/runtime/structured_turn.rs index ec7965747..0d0693109 100644 --- a/crates/cyber-server/src/runtime/structured_turn.rs +++ b/crates/cyber-server/src/runtime/structured_turn.rs @@ -2,13 +2,15 @@ //! output): a valid `structured_output` call ends the answer, a Turn that ends without //! one is asked again, and three failed submissions end it with `StructuredOutputError`. -use std::sync::Arc; +use std::{borrow::Cow, collections::BTreeMap, sync::Arc}; + +use cyber_store::NewEvent; use super::bus::LiveEvent; use super::drain::TurnEnd; use super::events::{SYSTEM_ADDED, SystemAdded, event}; -use super::structured::{MISSING_REASON, ROOT_ATTEMPTS, ROOT_TOOL, SETTLED}; -use super::{Handle, Inner, RuntimeError, StructuredOutcome}; +use super::structured::{MISSING_REASON, ROOT_ATTEMPTS, ROOT_TOOL, ResultState, SETTLED}; +use super::{Handle, Inner, RuntimeError, SessionState, StructuredOutcome, StructuredSchema}; const KIND: &str = "structured_output"; @@ -30,23 +32,14 @@ impl Inner { if self.closed.is_cancelled() || !state.result.root_open() { return Ok(()); } - let result = &state.result; - let value = result.returned.as_ref().map(|(_, value)| value.clone()); - let error = value.is_none().then(|| { - let why = match failure { - Some((kind, message)) => format!("{kind}: {message}"), - None if interrupted => "interrupted".into(), - None => "execution ended before structured_output".into(), - }; - format!("StructuredOutputError: {why}") - }); - let outcome = StructuredOutcome { - message_id: result.input.clone().expect("open root input"), - value, - error, - attempts: result.attempts, + let why = match failure { + Some((kind, message)) => format!("{kind}: {message}"), + None if interrupted => "interrupted".into(), + None => "execution ended before structured_output".into(), }; - self.commit_locked(&mut state, vec![event(SETTLED, &outcome)])?; + if let Some(outcome) = state.result.terminal_outcome(&why) { + self.commit_locked(&mut state, vec![event(SETTLED, &outcome)])?; + } Ok(()) } @@ -144,3 +137,34 @@ impl Inner { }) } } + +/// Freeze each replaced root input's outcome without changing live state. The +/// caller inserts it immediately before its replacement's promotion in one append. +pub(crate) fn promotion_outcomes( + state: &SessionState, + inputs: &[String], +) -> BTreeMap { + let mut outcomes = BTreeMap::new(); + if state.info.parent_id.is_some() { + return outcomes; + } + let mut previous = Cow::Borrowed(&state.result); + for input in inputs { + if let Some(outcome) = previous.terminal_outcome(&format!("superseded by input {input}")) { + outcomes.insert(input.clone(), event(SETTLED, &outcome)); + } + // Match on_promoted's schema selection for an intermediate input in this + // batch. Uncompilable historical schemas grant no new result ownership. + previous = Cow::Owned( + state + .inbox + .iter() + .find(|row| row.message_id == *input) + .and_then(|row| row.output_schema.clone()) + .and_then(|schema| StructuredSchema::new(schema).ok()) + .map(|schema| ResultState::root(schema, input.clone())) + .unwrap_or_default(), + ); + } + outcomes +} diff --git a/crates/cyber-server/tests/structured_root.rs b/crates/cyber-server/tests/structured_root.rs index d976c88ee..a393de327 100644 --- a/crates/cyber-server/tests/structured_root.rs +++ b/crates/cyber-server/tests/structured_root.rs @@ -3,6 +3,8 @@ //! `structured_output` finalizer, its corrections and limits, and its durable outcome. mod support; +#[path = "support/structured_terminal_fault.rs"] +mod terminal_fault; use cyber_server::runtime::{ Admission, Delivery, Runtime, RuntimeError, SessionState, StructuredOutcome, @@ -50,6 +52,275 @@ fn harness(main: Vec>) -> Harness { }) } +#[tokio::test] +async fn lost_terminal_commit_ack_reopens_the_original_outcome_without_dispatch() { + use std::sync::atomic::Ordering; + let h = harness(vec![structured("final", r#"{"summary":"committed"}"#)]); + let id = h.session().await; + let mut admission = Admission::text("audit", Delivery::Queue); + admission.output_schema = Some(schema()); + admission.resume = false; + let message = h.runtime.admit(&id, admission).await.unwrap().message_id; + let path = h.dir.path().join("cyber.db"); + let reached = terminal_fault::install(&h.store, &path, &id, &message); + assert!(!reached.load(Ordering::SeqCst)); + h.runtime.resume(&id).await.unwrap(); + h.settle(&id).await; + assert!( + reached.load(Ordering::SeqCst), + "the terminal outcome must actually commit" + ); + assert!( + h.store.has_failed(), + "lost COMMIT acknowledgement fences the writer" + ); + assert_eq!(h.models.requests("test/main").len(), 1); + assert!( + result_of(&h, &id, &message).await.is_none(), + "unacknowledged state is not published as success" + ); + let committed = input_event_seq(&h, &id, "session.structured_output.settled.1", &message); + tokio::time::timeout(std::time::Duration::from_secs(10), h.runtime.shutdown()) + .await + .unwrap(); + let (reopened_store, reopened) = reopened_runtime(&h); + for _ in 0..2 { + let state = reopened.state(&id).await.unwrap(); + let outcome = + outcome_from_state(&state, &message).expect("committed outcome survives reopen"); + assert_eq!(outcome.value, Some(json!({"summary":"committed"}))); + assert_eq!((outcome.error, outcome.attempts), (None, 1)); + } + assert_eq!( + h.models.requests("test/main").len(), + 1, + "lookup cannot redispatch" + ); + assert_eq!(reopened_store.aggregate_seq(&id).unwrap(), Some(committed)); + reopened.shutdown().await; +} + +fn reopened_runtime(h: &Harness) -> (std::sync::Arc, Runtime) { + let store = std::sync::Arc::new(open_store(&h.dir.path().join("cyber.db"))); + let replayed = runtime( + &store, + &h.models, + &h.tools, + h.dir.path(), + cyber_server::runtime::CompactionConfig::default(), + None, + ); + (store, replayed) +} + +#[tokio::test] +async fn a_lost_steering_promotion_ack_reopens_the_new_owner_without_replaying_the_old() { + use std::sync::atomic::Ordering; + let h = harness(vec![ + tools(&[("work", "read", "{}")]), + structured("final", r#"{"count":2}"#), + ]); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::Gated); + let id = h.session().await; + let started = h.tools.started.notified(); + let first = ask(&h, &id, "first", Some(schema())).await; + tokio::time::timeout(std::time::Duration::from_secs(10), started) + .await + .unwrap(); + let other = + json!({"type":"object","properties":{"count":{"type":"integer"}},"required":["count"]}); + let next = steer(&h, &id, "next", Some(other)).await; + let reached = terminal_fault::install(&h.store, &h.dir.path().join("cyber.db"), &id, &first); + h.tools.release.notify_one(); + h.settle(&id).await; + assert!(reached.load(Ordering::SeqCst)); + assert!(h.store.has_failed()); + assert_eq!( + h.models.requests("test/main").len(), + 1, + "failed writer must not admit the next request" + ); + assert_eq!(h.tools.executed.lock().unwrap().len(), 1); + assert!( + input_event_seq(&h, &id, "session.structured_output.settled.1", &first) + < input_event_seq(&h, &id, "session.prompt.promoted.1", &next) + ); + tokio::time::timeout(std::time::Duration::from_secs(10), h.runtime.shutdown()) + .await + .unwrap(); + let (store, replayed) = reopened_runtime(&h); + let before = replayed.state(&id).await.unwrap(); + let original = outcome_from_state(&before, &first).unwrap(); + assert_superseded(&original, 0, &next); + assert!(outcome_from_state(&before, &next).is_none()); + replayed.resume(&id).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), replayed.wait_idle(&id)) + .await + .unwrap(); + let after = replayed.state(&id).await.unwrap(); + assert_eq!(outcome_from_state(&after, &first), Some(original)); + let completed = outcome_from_state(&after, &next).unwrap(); + assert_eq!(completed.value, Some(json!({"count":2}))); + assert_eq!((completed.error, completed.attempts), (None, 1)); + assert_eq!(h.models.requests("test/main").len(), 2); + assert_eq!( + h.tools.executed.lock().unwrap().len(), + 1, + "recovery must not replay the old tool" + ); + let events = store.read_events(&id, -1, 200).unwrap().events; + assert_eq!( + events + .iter() + .filter(|event| event.kind == "session.prompt.promoted.1" + && event.data["message_id"] == next) + .count(), + 1 + ); + replayed.shutdown().await; +} + +async fn steer(h: &Harness, id: &str, text: &str, format: Option) -> String { + let mut admission = Admission::text(text, Delivery::Steer); + admission.output_schema = format; + admission.resume = false; + h.runtime.admit(id, admission).await.unwrap().message_id +} + +fn assert_superseded(outcome: &StructuredOutcome, attempts: u32, next: &str) { + assert_eq!(outcome.value, None); + assert_eq!(outcome.attempts, attempts); + assert_eq!( + outcome.error.as_deref(), + Some(format!("StructuredOutputError: superseded by input {next}").as_str()) + ); +} + +fn input_event_seq(h: &Harness, id: &str, kind: &str, message: &str) -> i64 { + h.store + .read_events(id, -1, 200) + .unwrap() + .events + .into_iter() + .find(|event| event.kind == kind && event.data["message_id"] == message) + .unwrap() + .seq +} + +#[tokio::test] +async fn steering_seals_each_displaced_result_before_promoting_its_replacement() { + let h = harness(vec![ + structured("invalid", r#"{"wrong":1}"#), + tools(&[("work", "read", "{}")]), + structured("new", r#"{"count":2}"#), + ]); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::Gated); + let id = h.session().await; + let started = h.tools.started.notified(); + let first = ask(&h, &id, "first", Some(schema())).await; + tokio::time::timeout(std::time::Duration::from_secs(10), started) + .await + .unwrap(); + let middle = steer(&h, &id, "middle", Some(schema())).await; + let other = + json!({"type":"object","properties":{"count":{"type":"integer"}},"required":["count"]}); + let last = steer(&h, &id, "last", Some(other.clone())).await; + assert!( + result_of(&h, &id, &first).await.is_none(), + "admission alone does not replace an open result" + ); + h.tools.release.notify_one(); + h.settle(&id).await; + let state = h.state(&id).await; + assert_superseded( + &outcome_from_state(&state, &first).expect("first outcome"), + 1, + &middle, + ); + assert_superseded( + &outcome_from_state(&state, &middle).expect("middle outcome"), + 0, + &last, + ); + let outcome = outcome_from_state(&state, &last).expect("last outcome"); + assert_eq!(outcome.value, Some(json!({"count":2}))); + assert_eq!((outcome.error, outcome.attempts), (None, 1)); + assert_eq!(h.models.requests("test/main").len(), 3); + assert!( + input_event_seq(&h, &id, "session.structured_output.settled.1", &first) + < input_event_seq(&h, &id, "session.prompt.promoted.1", &middle) + ); + assert!( + input_event_seq(&h, &id, "session.structured_output.settled.1", &middle) + < input_event_seq(&h, &id, "session.prompt.promoted.1", &last) + ); + let restarted = h.restart(); + let loaded = restarted.state(&id).await.unwrap(); + assert_eq!( + outcome_from_state(&loaded, &first), + outcome_from_state(&state, &first) + ); + assert_eq!( + outcome_from_state(&loaded, &middle), + outcome_from_state(&state, &middle) + ); + assert_eq!( + h.models + .requests("test/main") + .last() + .unwrap() + .tools + .iter() + .find(|tool| tool.name == "structured_output") + .unwrap() + .input_schema, + other + ); +} + +#[tokio::test] +async fn plain_steering_ends_the_prior_schema_without_applying_it_to_the_new_prompt() { + let h = harness(vec![tools(&[("work", "read", "{}")]), text("plain answer")]); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::Gated); + let id = h.session().await; + let started = h.tools.started.notified(); + let first = ask(&h, &id, "first", Some(schema())).await; + tokio::time::timeout(std::time::Duration::from_secs(10), started) + .await + .unwrap(); + let next = steer(&h, &id, "plain", None).await; + h.tools.release.notify_one(); + h.settle(&id).await; + assert_superseded( + &result_of(&h, &id, &first) + .await + .expect("superseded outcome"), + 0, + &next, + ); + assert!(result_of(&h, &id, &next).await.is_none()); + let requests = h.models.requests("test/main"); + assert_eq!(requests.len(), 2); + assert!( + requests[1] + .tools + .iter() + .all(|tool| tool.name != "structured_output") + ); +} + #[tokio::test] async fn a_durable_finalizer_reconciles_a_terminal_writer_cut_without_another_request() { assert_terminal_writer_cut(true).await; diff --git a/crates/cyber-server/tests/support/structured_terminal_fault.rs b/crates/cyber-server/tests/support/structured_terminal_fault.rs new file mode 100644 index 000000000..a04737b0d --- /dev/null +++ b/crates/cyber-server/tests/support/structured_terminal_fault.rs @@ -0,0 +1,67 @@ +//! Lose only the acknowledgement of the exact terminal-outcome commit. +use std::{ + cell::RefCell, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use cyber_store::Store; + +#[derive(Clone)] +struct Target { + database: PathBuf, + session: String, + message: String, + reached: Arc, +} + +// Every Store owns its writer thread; parallel fixtures cannot arm one another. +thread_local! { + static TARGET: RefCell> = const { RefCell::new(None) }; +} + +pub fn install(store: &Store, database: &Path, session: &str, message: &str) -> Arc { + let reached = Arc::new(AtomicBool::new(false)); + let target = Target { + database: database.to_owned(), + session: session.into(), + message: message.into(), + reached: reached.clone(), + }; + store + .transaction(move |connection| { + TARGET.with(|slot| *slot.borrow_mut() = Some(target)); + connection.wal_hook(Some(lose_terminal_ack)); + Ok(()) + }) + .unwrap(); + reached +} + +fn lose_terminal_ack(_: &rusqlite::hooks::Wal, _: i32) -> rusqlite::Result<()> { + let Some(target) = TARGET.with(|slot| slot.borrow().clone()) else { + return Ok(()); + }; + // WAL hooks run after commit. Read through a separate read-only connection, + // never recursively through Store or through a cached runtime projection. + let reader = rusqlite::Connection::open_with_flags( + &target.database, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + )?; + let committed: bool = reader.query_row( + "SELECT EXISTS(SELECT 1 FROM event WHERE aggregate_id=?1 AND type='session.structured_output.settled.1' AND json_extract(data,'$.message_id')=?2)", + rusqlite::params![target.session, target.message], |row| row.get(0), + )?; + if !committed { + return Ok(()); + } + TARGET.with(|slot| slot.borrow_mut().take()); + target.reached.store(true, Ordering::SeqCst); + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("injected structured terminal commit acknowledgement loss".into()), + )) +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 319658470..5a344a3a7 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -22,7 +22,7 @@ section 5/6 implementation state; reconcile it at delivery. | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill acceptance passes locally, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | | #93 rules/context, tasks 4.1–4.4 | Implement trusted path-rule discovery/precedence/bounds and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | -| #93 structured results, tasks 5.1–5.4 | #116 merged admission/schema bounds, finalization, retries, per-input outcomes and CLI/API/SDK rendering. Runtime acceptance proves compaction, terminal interruption/budget outcomes, shutdown/resume and recovery from a failed terminal append before queued schema promotion. Real CLI budget/timeout exit 4 and Unix SIGINT exit 130 pass locally. Finish post-commit acknowledgement-loss and steering ownership acceptance plus current-source platform checks; lookup after completed prompts alone does not prove those cuts. | +| #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | | #92 protocol/callbacks/producers, tasks 3–6/S/D | Finish contribution ownership, hooks/providers/catalog transforms and TypeScript kit, durable scoped callbacks (`session/prompt` still explicitly refuses), Setup and DirectoryAdded plus combined producer acceptance. Preserve accepted FileChanged and CwdChanged fixes. Complete S1 epoch/directory-key/legacy-deletion and D1 directory-admission/Safe Boundary/child-policy review before their dependent code. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 2032fbc8f..c8ca85b09 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -636,6 +636,7 @@ cyber exec --output-schema findings.schema.json --format json "audit the depende - **Corrections:** an invalid call returns the validation errors to the model, and an answer that ends without the call gets a reminder. Three failed submissions in all (the first plus two corrections) end the prompt with `StructuredOutputError: ` and exit code 1; there is never a fourth. - **Interrupted attempts:** once a schema-governed input is promoted, an interruption or budget stop records its terminal error and current submission count before the Drain becomes idle. Lookup after restart reads that outcome without dispatching another request. Shutdown preserves an unfinished attempt for explicit resume; compaction preserves its schema and retry count. - **Recovering a terminal write:** if the finalizer or exhausted submissions are durable but their terminal-outcome append failed, resume first settles that unfinished result without another provider request. Any queued new prompt then uses its own schema and attempt count. +- **Steering:** promoting a replacement input closes an unfinished structured result with `StructuredOutputError: superseded by input …`, preserving its submission count. An already valid finalizer keeps its value. The outcome commits together with the replacement promotion; intermediate schema inputs in one steering batch can finish with zero submissions. A plain replacement does not inherit the old schema. - **Output:** text mode prints the object as pretty JSON; `--format json` and the `stream-json` `result` line put it in `result` (`null` on failure, with the error in `error`). Interruption (130), budgets and timeouts (4) keep their usual exit codes. - **API:** `POST /api/v1/sessions/{id}/prompt` takes `"format": {"type": "json_schema", "schema": {…}}` on a root Session's prompt (a subagent has its own result schema). The schema belongs to that prompt only: queued prompts keep their own schemas, results and counts, also across a restart. The outcome is recorded as `session.structured_output.settled.1` (`message_id`, `value` or `error`, `attempts`) and kept on the input's inbox row as `structured`, so a client that lost the reply can read it again. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md index 06dc5a914..4c8357dfd 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/session-runtime/spec.md @@ -30,3 +30,14 @@ - **WHEN** an explicit resume recovers that unfinished result after restart - **THEN** it SHALL settle the original value or exhausted error before model dispatch, preserving its exact input and attempt count without another provider request - **AND** any new queued prompt SHALL be promoted only after that reconciliation, retaining its own schema and result; ordinary result lookup SHALL never redispatch work + +#### Scenario: Steering replaces an unfinished structured input +- **WHEN** a Safe Boundary promotes another root input while a structured result remains open +- **THEN** the previous input SHALL retain its durable valid finalizer or receive a terminal `StructuredOutputError` naming the superseding input, with its original submission count +- **AND** that outcome and the replacement promotion SHALL commit atomically in order; several steered schema inputs promoted together SHALL each retain their own outcome, including zero-submission superseded inputs +- **AND** a new unstructured input SHALL not inherit the previous schema, and a different schema SHALL not borrow the previous result or retry count + +#### Scenario: Terminal or steering commit acknowledgement is lost +- **WHEN** SQLite commits a terminal outcome or outcome/promotion batch but its commit acknowledgement fails +- **THEN** the failed writer SHALL remain fenced and cached terminal state SHALL NOT be advanced on an unacknowledged reply +- **AND** reopening the physical database SHALL recover the exact committed outcome and attempt count without inference; a committed steering promotion SHALL resume only its new input without replaying the displaced input or tool diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 57be72597..94a422bd0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -137,14 +137,14 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - [x] 5.1 Add validated per-prompt root format admission, bounded schema compilation and durable schema/attempt ownership without changing existing child contracts. - [x] 5.2 Add `structured_output` finalization, two correction retries, budget/interrupt handling and durable successful/error result settlement. - [x] 5.3 Add `--output-schema`, API/SDK result fields and text/JSON/stream rendering; preserve unstructured commands and validate option combinations before admission. -- [ ] 5.4 Test valid/invalid schemas, malformed results, retry exhaustion, missing finalizer, compaction, restart/retry identity, budget exits and independent queued prompts with a real server/scripted provider. Done: schema bounds and refusal before admission, malformed and invalid submissions, exhaustion without a fourth request, missing finalizer, restart between independent queued prompts, retry identity with another schema, runtime compaction during correction, durable interruption/budget outcomes, shutdown/resume of an open result, failed terminal-append recovery before another dispatch/queued schema promotion, and real CLI budget/timeout exit 4 and Unix SIGINT exit 130. Open: post-commit terminal acknowledgement loss, steering ownership and current-source platform acceptance. +- [ ] 5.4 Test valid/invalid schemas, malformed results, retry exhaustion, missing finalizer, compaction, restart/retry identity, budget exits and independent queued prompts with a real server/scripted provider. Local functional cases pass: schema bounds, admission refusal, malformed/invalid submissions, retry exhaustion, missing finalizer, compaction during correction, independent queued schemas, shutdown/resume, interruption/budget outcomes, terminal writer cuts, post-commit acknowledgement loss, steering ownership, and CLI budget/timeout exit 4 and Unix SIGINT exit 130. Current-source platform acceptance remains required; this task is not accepted from local results alone. ## 6. Streaming JSONL input - [x] 6.1 Define typed version-2 envelopes, correlation, resource bounds, aggregate exit rules and driver-owned interactive request lifetime; document compatibility and unsupported combinations. - [ ] 6.2 Implement concurrent bounded parsing/admission and exact owned permission/question replies, retained request lookup, interrupt and end/EOF draining without stopping unrelated work. Basic parsing/admission merged in #117. The 2026-11-09 code review found that request collection and interrupt target the Session without checking the driver's admitted message ownership, and completed inputs retain their full result payloads indefinitely. Complete those ownership/resource bounds and their shared-Session/long-run regressions before accepting this task. - [ ] 6.3 Emit per-input results, optional partial/hook events and atomic last-message output with bounded backpressure and secret-safe errors. `--include-hook-events` is not implemented: hook receipts are deliberately not on the Session stream, so the lines would have nothing reliable to carry. -- [ ] 6.4 Test multiple inputs, steering/queueing, malformed/oversized input, capacity, stale/cross-request replies, EOF while awaiting input, slow stdout, SIGINT, server restart and lost admission acknowledgement. Done: multiple inputs in order, per-input structured results, malformed input (exit 2), unknown request replies, EOF while a request waits, unsupported combinations, with the real binary (`cyber-cli/tests/exec_stream_input.rs`). Open: steering, capacity, slow stdout, SIGINT, server restart and lost admission acknowledgement. +- [ ] 6.4 Test multiple inputs, steering/queueing, malformed/oversized input, capacity, stale/cross-request replies, EOF while awaiting input, slow stdout, SIGINT, server restart and lost admission acknowledgement. Done: multiple inputs in order, per-input structured results, structured/plain steering while inference is gated, long-run slot retirement, malformed input (exit 2), unknown request replies, EOF while a request waits, unsupported combinations, with the real binary (`cyber-cli/tests/exec_stream_input.rs`). Open: shared-Session ownership, capacity, slow stdout, SIGINT, server restart and lost admission acknowledgement. ## 7. Review and delivery @@ -817,3 +817,33 @@ This edit is not current-source external acceptance: run 42693044871 remains on 9ac06c40 (Linux x64/ARM passed; macOS jobs queued at the preceding audit). No new API/SDK field or CLI flag was added. SDK/plugin/full-workspace checks and Windows execution were not rerun here; the SIGINT case is explicitly Unix-only. + +Structured steering and committed-reply continuation (2026-11-09): two new +runtime regressions first failed because a steered promotion discarded the prior +input's open result. Promotion now freezes displaced outcomes and commits each +immediately before its replacement in the same append. A valid durable finalizer +wins; otherwise each displaced structured input gets an explicit superseded +error with its own attempt count, including zero-attempt intermediate schemas. +Child result behavior and unstructured input semantics are unchanged. A shared +pure result constructor also preserves the same terminal rules at Drain exit. + +Runtime cases prove multiple steered schemas and replacement by plain input. +Two real JSONL cases gate inference, admit steering concurrently over stdin, and +verify exact per-input outcomes plus aggregate failure for a superseded schema. +Two SQLite WAL-hook cases lose the acknowledgement only after the exact target +terminal commit is visible through an independent read-only connection. They +prove the writer fences, cached terminal state does not advance, and reopening +the physical Store recovers the original result or committed new input without +replaying old inference/tools or duplicating promotion. The native workflow now +watches the fault helper. Manual proof dispatches queue behind an active matrix +rather than cancelling it; obsolete pull-request heads still cancel normally. + +Checks: 115 broad runtime/CLI regressions pass, followed by both added WAL-ack +cases. The final targeted run passes all 16 structured-root, five output-schema +CLI and 11 JSONL tests; combined server/CLI all-target Clippy passes with warnings +denied. Formatting, whitespace and strict OpenSpec pass (100 items; lint zero +errors/25 warnings). Focused code/security review confirms state stays unchanged +until the atomic promotion append succeeds and no request/reply or interruption +authority is inferred from these result records. No new API/SDK fields were +added. Current-source native/platform acceptance and global SDK/plugin/full +workspace checks remain open. This does not complete tasks 6.2–6.4 or M2–M8. From 75d8459def4d1ec8995a4c741c302bd115875613 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 20:42:35 -0400 Subject: [PATCH 58/87] Reclaim cancelled prepared Git snapshot payloads --- .github/workflows/non-git-snapshot-proof.yml | 4 +- crates/cyber-cli/tests/branches_cli.rs | 236 +++++++++++++++--- crates/cyber-snapshot/src/copy/reclamation.rs | 6 +- .../src/copy/reclamation/cancelled.rs | 9 +- docs/goal-delivery.md | 2 +- .../managed-recovery.md | 12 +- .../specs/snapshots-checkpoints/spec.md | 9 +- .../complete-daily-coding-surfaces/tasks.md | 39 +++ 8 files changed, 266 insertions(+), 51 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 7255a8619..199554708 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -77,11 +77,11 @@ jobs: set -euo pipefail cargo test --locked -p cyber-snapshot --features native-git-acceptance -- --nocapture 2>&1 | tee git-adapter-proof.log cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_git_runtime -- --nocapture 2>&1 | tee git-runtime-proof.log - - name: Managed Git CLI process-death recovery candidate + - name: Managed Git CLI process-death recovery and cancellation candidate shell: bash run: | set -euo pipefail - cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_git_keep_and_rollback_after_real_partial_restore_kill -- --nocapture 2>&1 | tee git-cli-recovery-proof.log + cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_git_ -- --nocapture --test-threads=1 2>&1 | tee git-cli-recovery-proof.log - name: JSONL driver and atomic last-message regressions shell: bash run: | diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 27e867b1c..41352be46 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -809,10 +809,7 @@ fn managed_git_keep_and_rollback_after_real_partial_restore_kill() { } } -#[cfg(all( - feature = "native-git-acceptance", - any(target_os = "macos", target_os = "linux") -))] +#[cfg(any(target_os = "macos", target_os = "linux"))] fn git_fixture(directory: &std::path::Path, args: &[&str]) -> String { let output = Command::new("git") .arg("-C") @@ -828,9 +825,8 @@ fn git_fixture(directory: &std::path::Path, args: &[&str]) -> String { feature = "native-git-acceptance", any(target_os = "macos", target_os = "linux") ))] -fn managed_git_partial_kill(action: &str) { - let model = scripted_model::ScriptedModel::start(reply); - let mut env = Env::new(&model); +fn managed_git_fixture(model: &scripted_model::ScriptedModel) -> (Env, String) { + let mut env = Env::new(model); git_fixture( &env.directory, &[ @@ -863,6 +859,16 @@ fn managed_git_partial_kill(action: &str) { env.directory .starts_with(env.root.join("cyber-home/data/worktrees")) ); + (env, id) +} + +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn managed_git_partial_kill(action: &str) { + let model = scripted_model::ScriptedModel::start(reply); + let (env, id) = managed_git_fixture(&model); for index in 0..1280 { std::fs::write(env.directory.join(format!("bulk_{index:04}")), "before").unwrap(); } @@ -898,7 +904,9 @@ fn managed_git_partial_kill(action: &str) { let (journal, review) = reviewed_recovery_cli(&env, &id, action); assert_eq!( review["origin"]["worktree_id"], - created["data"]["worktree"]["id"] + cyber_core::worktrees::Repository::managed_locations_at(&env.directory).unwrap()[0] + .1 + .id ); let request_id = format!("managed-git-kill-{action}"); let request = json!({"journal_id":journal,"action":action,"confirmation":review["confirmation"],"request_id":request_id}); @@ -1095,6 +1103,18 @@ fn fully_prepared_pre_effect_cancellation_reclaims_only_persisted_target_inodes( cancelled_preparation(true, false, true); } +#[test] +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn managed_git_prepared_cancellation_reclaims_persisted_files_and_dangling_links() { + let model = scripted_model::ScriptedModel::start(reply); + let (env, id) = managed_git_fixture(&model); + let fixture = preparation_in(model, env, Some(id), true); + run_cancelled_preparation(fixture, true, false, true); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] struct PreparationFixture { model: scripted_model::ScriptedModel, @@ -1108,18 +1128,40 @@ struct PreparationFixture { fn preparation_fixture(prepared: bool) -> PreparationFixture { let model = scripted_model::ScriptedModel::start(reply); let env = Env::new(&model); - let repo = env.root.join("repo"); - std::fs::remove_dir_all(repo.join(".git")).unwrap(); - let before = format!("b\0{}", "b".repeat(64 * 1024)); - let after = format!("a\0{}", "a".repeat(64 * 1024)); + std::fs::remove_dir_all(env.directory.join(".git")).unwrap(); + preparation_in(model, env, None, prepared) +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn preparation_in( + model: scripted_model::ScriptedModel, + env: Env, + session: Option, + prepared: bool, +) -> PreparationFixture { + let repo = env.directory.clone(); + let git = repo.join(".git").exists(); + // Git hands off several images per path under its bounded import budget. + // Keep separate flushed staging operations without inflating that handoff. + let bytes = if git { 8 * 1024 } else { 64 * 1024 }; + let before = format!("b\0{}", "b".repeat(bytes)); + let after = format!("a\0{}", "a".repeat(bytes)); for index in 0..1280 { std::fs::write(repo.join(format!("bulk_{index:04}")), &before).unwrap(); } - let id = env.exec(None, "before snapshot"); + if git { + std::os::unix::fs::symlink("bulk_0000", repo.join("a_link")).unwrap(); + git_fixture(&repo, &["add", "--all"]); + } + let id = env.exec(session.as_deref(), "before snapshot"); let first = env.user_ids(&id)[0].clone(); for index in 0..1280 { std::fs::write(repo.join(format!("bulk_{index:04}")), &after).unwrap(); } + if git { + std::fs::remove_file(repo.join("a_link")).unwrap(); + std::os::unix::fs::symlink("bulk_1279", repo.join("a_link")).unwrap(); + } if prepared { std::fs::write(repo.join("aa_removed"), "after-only source").unwrap(); } @@ -1137,6 +1179,16 @@ fn preparation_fixture(prepared: bool) -> PreparationFixture { #[cfg(any(target_os = "macos", target_os = "linux"))] fn cancelled_preparation(published: bool, terminal: bool, prepared: bool) { + run_cancelled_preparation(preparation_fixture(prepared), published, terminal, prepared); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn run_cancelled_preparation( + fixture: PreparationFixture, + published: bool, + terminal: bool, + prepared: bool, +) { let PreparationFixture { model, env, @@ -1144,8 +1196,10 @@ fn cancelled_preparation(published: bool, terminal: bool, prepared: bool) { first, messages, after, - } = preparation_fixture(prepared); - let repo = env.root.join("repo"); + } = fixture; + let repo = &env.directory; + let git_metadata = preparation_git_metadata(repo); + let source_link = std::fs::read_link(repo.join("a_link")).ok(); let activity = if prepared { kill_after_prepared_images(&env, &id, &first) } else { @@ -1176,7 +1230,7 @@ fn cancelled_preparation(published: bool, terminal: bool, prepared: bool) { let receipt = preparation_with_client(&env, &id, &activity, &body, terminal); let request_id = receipt["request_id"].as_str().unwrap(); let body = json!({"confirmation":receipt["confirmation"],"request_id":request_id}); - assert_preparation_resource_digest(&review, &receipt, published); + assert_preparation_resource_digest(&env, &review, &receipt, published); assert_eq!( json(&preparation_request_cli( &env, &id, &activity, request_id, true @@ -1187,25 +1241,51 @@ fn cancelled_preparation(published: bool, terminal: bool, prepared: bool) { assert_preparation_tombstone(&env, &receipt); } assert_preparation_receipt_restart(&env, &receipt, &body); + assert_cancelled_preparation_source(&env, &after); + assert_eq!(env.messages(&id), messages); + assert_eq!(model.requests().len(), requests); + if prepared { + assert_preparation_cleanup_lookup(&env, &receipt); + } + assert_eq!(env.messages(&id), messages); + assert_eq!(model.requests().len(), requests); + assert_eq!(std::fs::read_link(repo.join("a_link")).ok(), source_link); + assert_cancelled_preparation_source(&env, &after); + assert_eq!(preparation_git_metadata(repo), git_metadata); + env.exec(Some(&id), "after preparation cancellation"); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_cancelled_preparation_source(env: &Env, after: &str) { assert!( - std::fs::symlink_metadata(repo.join("bulk_0000")) + std::fs::symlink_metadata(env.directory.join("bulk_0000")) .unwrap() .file_type() .is_symlink() ); assert_eq!( - std::fs::read_to_string(target).unwrap(), + std::fs::read_to_string(env.root.join("preparation-owner-target")).unwrap(), "later owner bytes" ); - assert_eq!(env.file("bulk_1279"), Some(after)); - assert_eq!(env.messages(&id), messages); - assert_eq!(model.requests().len(), requests); - if prepared { - assert_preparation_cleanup_lookup(&env, &receipt); + assert_eq!(env.file("bulk_1279").as_deref(), Some(after)); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn preparation_git_metadata(directory: &std::path::Path) -> Vec<(PathBuf, Vec)> { + if !directory.join(".git").exists() { + return Vec::new(); } - assert_eq!(env.messages(&id), messages); - assert_eq!(model.requests().len(), requests); - env.exec(Some(&id), "after preparation cancellation"); + ["index", "config"] + .into_iter() + .map(|name| { + let path = PathBuf::from(git_fixture( + directory, + &["rev-parse", "--path-format=absolute", "--git-path", name], + )); + let bytes = std::fs::read(&path).unwrap(); + (path, bytes) + }) + .collect() } #[cfg(any(target_os = "macos", target_os = "linux"))] @@ -1311,10 +1391,13 @@ fn assert_prepared_payload_cleanup(env: &Env, reply: &Value) { .join(&receipt.origin.as_ref().unwrap().binding.journal_id); let intent = std::fs::read(journal.join("intent.json")).unwrap(); let cancellation = std::fs::read(journal.join("preparation.json")).unwrap(); - let source = env.root.join("repo"); + let source = env.directory.clone(); assert_eq!(maintenance.reclaim(&source).unwrap(), 0); maintenance.age(receipt.activity_id.clone()); + #[cfg(feature = "native-git-acceptance")] + assert_prepared_managed_guards(env, &maintenance, &receipt); let receipt = Receipt::Preparation(Box::new(receipt)); + assert_prepared_link_guards(&maintenance, &source, &journal, &receipt); assert_prepared_inode_and_lifetime_guards(&maintenance, &source, &journal, &receipt); assert_prepared_store_receipt_binding(&maintenance, &source, &journal, &receipt); assert_prepared_interrupted_plan(&maintenance, &source, &journal, &receipt); @@ -1346,6 +1429,38 @@ fn assert_prepared_payload_cleanup(env: &Env, reply: &Value) { ); } +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn assert_prepared_managed_guards( + env: &Env, + maintenance: &reclamation_support::Maintenance, + receipt: &cyber_server::runtime::SnapshotPreparationReceipt, +) { + if receipt.lifetime.owner.worktree_id.is_none() { + return; + } + let phase: Value = serde_json::from_slice( + &std::fs::read( + maintenance + .namespace + .join("workers") + .join(receipt.lifetime.source_phase.as_ref().unwrap()), + ) + .unwrap(), + ) + .unwrap(); + let bindings: Vec = + serde_json::from_value(phase["checkout_bindings"].clone()).unwrap(); + assert!(!bindings.is_empty()); + let owner = receipt.lifetime.checkout_owner().unwrap(); + let receipt = cyber_server::runtime::snapshot_reclamation::Receipt::Preparation(Box::new( + receipt.clone(), + )); + assert_managed_cleanup_refusals(env, maintenance, &receipt, &bindings[0], &owner); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_prepared_interrupted_plan( maintenance: &reclamation_support::Maintenance, @@ -1362,7 +1477,7 @@ fn assert_prepared_interrupted_plan( std::fs::write(&unknown, b"owner bytes").unwrap(); assert!(maintenance.reclaim(source).is_err()); let first = journal.join(&pending.plan.files[0].name); - assert!(first.exists()); + assert!(std::fs::symlink_metadata(&first).is_ok()); std::fs::remove_file(unknown).unwrap(); // Synthetic progress of one planned private unlink, following a real kill. std::fs::remove_file(first).unwrap(); @@ -1420,7 +1535,7 @@ fn assert_prepared_inode_and_lifetime_guards( held.try_lock().unwrap(); assert!(maintenance.reclaim(source).is_err()); drop(held); - let stage = journal.join("stage_1"); + let stage = prepared_stage(journal, "bulk_0000"); let retained = journal.join("retained-original"); let bytes = std::fs::read(&stage).unwrap(); std::fs::rename(&stage, &retained).unwrap(); @@ -1432,6 +1547,43 @@ fn assert_prepared_inode_and_lifetime_guards( std::fs::rename(retained, stage).unwrap(); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn prepared_stage(journal: &std::path::Path, path: &str) -> PathBuf { + let intent: Value = + serde_json::from_slice(&std::fs::read(journal.join("intent.json")).unwrap()).unwrap(); + let index = intent["changes"] + .as_array() + .unwrap() + .iter() + .position(|change| change["path"] == path) + .unwrap(); + journal.join(format!("stage_{index}")) +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_prepared_link_guards( + maintenance: &reclamation_support::Maintenance, + source: &std::path::Path, + journal: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + if !source.join(".git").exists() { + return; + } + let stage = prepared_stage(journal, "a_link"); + let target = std::fs::read_link(&stage).unwrap(); + assert_eq!(target, std::path::Path::new("bulk_0000")); + assert!(!stage.exists(), "private prepared target must be dangling"); + let retained = journal.join("retained-link"); + std::fs::rename(&stage, &retained).unwrap(); + std::os::unix::fs::symlink(&target, &stage).unwrap(); + let error = maintenance.reclaim(source).unwrap_err(); + assert!(error.contains("persisted ownership"), "{error}"); + assert!(maintenance.policy.record(receipt).unwrap().is_none()); + std::fs::remove_file(&stage).unwrap(); + std::fs::rename(retained, stage).unwrap(); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn terminal_preparation(env: &Env, session: &str, activity: &str) -> Value { let template = env.command(&[]); @@ -1582,7 +1734,7 @@ fn assert_preparation_client_refusals(env: &Env, session: &str, activity: &str, } #[cfg(any(target_os = "macos", target_os = "linux"))] -fn assert_preparation_resource_digest(review: &Value, receipt: &Value, published: bool) { +fn assert_preparation_resource_digest(env: &Env, review: &Value, receipt: &Value, published: bool) { assert_eq!( receipt["checkout_resources_digest"], review["checkout_resources_digest"] @@ -1590,9 +1742,31 @@ fn assert_preparation_resource_digest(review: &Value, receipt: &Value, published if published { let lifetime: cyber_server::runtime::SnapshotLifetimeBinding = serde_json::from_value(review["lifetime"].clone()).unwrap(); + let namespace = std::fs::read_dir(env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.join("workers") + .join(&lifetime.owner.activity_id) + .exists() + }) + .unwrap(); + let phase: Value = serde_json::from_slice( + &std::fs::read( + namespace + .join("workers") + .join(lifetime.source_phase.as_ref().unwrap()), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(phase["lifetime"], review["lifetime"]); + let bindings: Vec = + serde_json::from_value(phase["checkout_bindings"].clone()).unwrap(); + assert_eq!(bindings.is_empty(), lifetime.owner.worktree_id.is_none()); assert_eq!( receipt["checkout_resources_digest"], - lifetime.checkout_resources_digest(&[]).unwrap() + lifetime.checkout_resources_digest(&bindings).unwrap() ); } } diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index 9b5ed686c..dcc3367b7 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -2,11 +2,7 @@ mod cancelled; mod checkouts; mod paired; -use super::{ - Store, ownership, - restore::{Journal, JournalBackend}, - storage, -}; +use super::{Store, ownership, restore::Journal, storage}; use crate::reclamation::{Policy, Settlement}; use cap_fs_ext::DirExt; use cap_std::fs::Dir; diff --git a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs index 5a8ab7146..05f5410fe 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs @@ -10,11 +10,7 @@ pub(super) fn eligible(record: &Journal) -> bool { && !record.acknowledged && record.settled_by.is_none() && (!record.completed || record.changes.is_empty()) - && record - .context - .as_ref() - .is_some_and(|context| context.git.is_none()) - && record.backend != Some(JournalBackend::Git) + && record.context.is_some() } pub(super) fn verify( store: &Store, @@ -172,8 +168,7 @@ fn payloads( let previous = planned.get(name.as_str()).copied(); match (&change.after, image) { (Some(after), Some(image)) - if after.kind == super::super::ImageKind::File - && image.entry == *after + if image.entry == *after && ownership.prepared_identity(index) == Some(image.identity.as_str()) => { files.push(Payload { diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 5a344a3a7..2554b36d9 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -20,7 +20,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | -| #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill acceptance passes locally, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | +| #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | | #93 rules/context, tasks 4.1–4.4 | Implement trusted path-rule discovery/precedence/bounds and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index beea6c010..f9d1759c6 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -161,7 +161,7 @@ cleanup visibly. No direct user removal of private journal directories is requir ## Managed checkout payload reclamation -Reclamation of acknowledged regular-file Git journal payloads SHALL reacquire +Reclamation of acknowledged regular-file and symlink Git journal payloads SHALL reacquire the complete persisted checkout set for each consumed worker lifetime before preparing a durable deletion plan. Each claim SHALL bind the exact original creation and lifetime, exclude a live worker, and retain an already published @@ -180,7 +180,7 @@ native evidence SHALL remain unchanged. Restart after a durable plan SHALL use the same frozen payload identities and complete-resource proof, never source replay. Existing receipt and reclamation event formats remain unchanged. -Typed link payloads, incomplete construction, legacy missing proof, CAS/metadata +Incomplete construction, legacy missing proof, CAS/metadata collection and the complete platform/cut matrix remain separate obligations. This extension does not enable default owned Git restore admission. @@ -942,8 +942,12 @@ and be rechecked during deletion; unknown native owners continue to protect both ### Cancelled prepared-image cleanup An exact preparation cancellation with a published origin, positive non-admitted -version-2 phase, complete empty checkout set and matching native cancellation -metadata MAY reclaim fully prepared regular target images. Each leaf SHALL match +version-2 phase, complete persisted checkout set and matching native cancellation +metadata MAY reclaim fully prepared regular-file and symlink target images in +file-copy or Git journals. Managed checkouts SHALL use the same retained, +already-published acknowledgement requirements as ordinary and paired cleanup. +Symlink observations and unlinks SHALL retain the link inode without following +its target, including dangling private targets. Each leaf SHALL match its persisted prepared inode identity and immutable target image; incomplete construction without those identities SHALL remain protected. The version-3 plan SHALL freeze the native intent digest and complete private inventory before diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 331c1e605..6b306ee50 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -478,12 +478,19 @@ - **AND** repeated request lookup and explicit metadata acknowledgement after restart SHALL NOT replay source changes or start inference #### Scenario: Reclaiming acknowledged managed Git payloads -- **GIVEN** an aged acknowledged Git restore or recovery pair has exact committed receipts, persisted checkout sets and regular-file payload ownership +- **GIVEN** an aged acknowledged Git restore or recovery pair has exact committed receipts, persisted checkout sets and regular-file or symlink payload ownership - **WHEN** maintenance prepares and consumes a durable reclamation plan - **THEN** it SHALL retain every exited checkout claim and already published acknowledgement matching the exact lifetime, resource digest and consuming receipt through deletion - **AND** a missing acknowledgement, live claim or changed registration SHALL refuse before plan publication without repairing metadata - **AND** restart after plan publication SHALL use frozen native payload identities without source replay, preserving original claims, receipts and user Git metadata +#### Scenario: Reclaiming cancelled fully prepared Git images +- **GIVEN** a fully prepared Git journal has an exact committed cancellation, positive non-admitted phase, persisted target identities and complete consumed checkout evidence +- **WHEN** retention permits maintenance to plan private payload cleanup +- **THEN** it SHALL require every managed checkout's already published exact acknowledgement and retain the native claims through deletion +- **AND** regular files and dangling symlinks SHALL match immutable target images and prepared native identities without following link targets +- **AND** replacement inodes, missing markers, live claims and unknown entries SHALL refuse; restart after a durable plan SHALL preserve source files, user Git metadata and cancellation lookup without replay + #### Scenario: Acknowledged private payload is a symbolic link - **GIVEN** an ordinary restore or settled recovery pair has immutable typed link images and exact private inode ownership - **WHEN** reference-aware maintenance prepares or resumes its receipt-bound cleanup plan diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 94a422bd0..15c5f95f7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -847,3 +847,42 @@ until the atomic promotion append succeeds and no request/reply or interruption authority is inferred from these result records. No new API/SDK fields were added. Current-source native/platform acceptance and global SDK/plugin/full workspace checks remain open. This does not complete tasks 6.2–6.4 or M2–M8. + + +Cancelled fully prepared Git payload continuation (2026-11-09): cancellation +cleanup now accepts fully persisted Git target images and typed symlinks through +the same version-3 intent-bound plan, positive non-admitted phase, complete checkout +set and exact already-published acknowledgement checks as other cleanup classes. +No source replay, receipt field, plan format or default Git admission is added. +Incomplete construction without persisted identities, CAS/metadata collection and +the remaining native/cut/client gates remain open with M6 and its parent tasks. + +Both real CLI prepared-cancellation cases pass on local macOS ARM (235.48 seconds): +file-copy and managed Git. The managed case kills the real service after prepared +identities are published while SQLite writer contention prevents source admission. +It verifies non-empty persisted checkout digest evidence, missing-marker and live +claim refusal before a plan, equal-target symlink and regular-file replacement +refusal, dangling private link ownership, immutable Store intent binding, unknown +entry refusal, resumed cleanup after a durable plan and synthetic planned unlink, +retained cancellation lookup after restart, unchanged owner files/transcript/Git +index/configuration and no cleanup inference. The Git fixture uses smaller images +under the unchanged handoff limit; its first oversized attempt was rejected before +cleanup, and an empty-resource assertion inherited from the file-copy fixture was +corrected to verify the actual persisted bindings. Neither failed attempt counts +as acceptance. The final helper refactor and persisted-set checks are in the passing +binary. Snapshot library: 140 passed, six intentionally ignored subprocess fixtures. +All-target snapshot/CLI Clippy with warnings denied, formatting/whitespace and +strict OpenSpec pass (100 items; lint zero errors/25 warnings). The compiler still +reports macOS sandbox deprecation and linker unwind-table warnings. Focused code +and security review found no additional issue in this extension; this is not a +review of the full recovery branch. SDK/generated-artifact, default CLI, full +workspace and current-source external platform acceptance were not rerun here. +The native workflow now selects both managed Git process-death cases serially; +the workflow edit is not passing platform evidence. + +Native run 42716226637 at 03b4526a completed successfully on Linux x64/ARM and +macOS ARM/Intel. It predates steering and this cancellation extension. Run +42728260672 at 1be03642 covers steering and post-commit acknowledgement loss; +its Linux x64 job has passed, with the remaining jobs pending at this checkpoint. +Neither run accepts the new cancelled-Git cleanup source. Queue its own native +proof and retain each run's actual tested source identity. From e2b3ba181bbd7a10d982b18fcb8f533d14d27eff Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Mon, 9 Nov 2026 21:09:52 -0400 Subject: [PATCH 59/87] Honor fallback snapshot retention during collection --- .github/workflows/non-git-snapshot-proof.yml | 20 ++ .../cyber-app/tests/snapshot_reclamation.rs | 83 ++++++ crates/cyber-cli/tests/branches_cli.rs | 20 ++ crates/cyber-snapshot/src/copy/collection.rs | 253 ++++++++++++++++++ crates/cyber-snapshot/src/copy/lifetime.rs | 10 +- crates/cyber-snapshot/src/copy/mod.rs | 141 +--------- .../src/copy/reclamation/paired.rs | 34 ++- crates/cyber-snapshot/tests/file_copy.rs | 231 ++++++++++++++++ docs/goal-delivery.md | 12 + .../managed-recovery.md | 9 + .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 47 ++++ 12 files changed, 723 insertions(+), 144 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/collection.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 199554708..01b40b4ad 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -15,6 +15,7 @@ on: - crates/cyber-server/tests/support/structured_terminal_fault.rs - crates/cyber-server/src/http/sessions.rs - crates/cyber-app/tests/snapshot_git_runtime.rs + - crates/cyber-app/tests/snapshot_reclamation.rs - crates/cyber-cli/Cargo.toml - crates/cyber-cli/src/** - crates/cyber-cli/tests/branches_cli.rs @@ -77,18 +78,28 @@ jobs: set -euo pipefail cargo test --locked -p cyber-snapshot --features native-git-acceptance -- --nocapture 2>&1 | tee git-adapter-proof.log cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_git_runtime -- --nocapture 2>&1 | tee git-runtime-proof.log + - name: Store-bound fallback collection and reclamation + shell: bash + run: | + set -euo pipefail + cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_reclamation -- --nocapture 2>&1 | tee fallback-collection-proof.log - name: Managed Git CLI process-death recovery and cancellation candidate + env: + TMPDIR: ${{ runner.temp }}/snapshot-fixtures shell: bash run: | set -euo pipefail + mkdir -p "$TMPDIR" cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_git_ -- --nocapture --test-threads=1 2>&1 | tee git-cli-recovery-proof.log - name: JSONL driver and atomic last-message regressions + if: ${{ !cancelled() }} shell: bash run: | set -euo pipefail cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::exec:: -- --nocapture 2>&1 | tee jsonl-driver-proof.log cargo test --locked -p cyber-cli --features native-git-acceptance --test exec_stream_input -- --nocapture 2>&1 | tee -a jsonl-driver-proof.log - name: Root structured result recovery and CLI stop regressions + if: ${{ !cancelled() }} shell: bash run: | set -euo pipefail @@ -103,7 +114,16 @@ jobs: managed-checkout-ack-proof.log git-adapter-proof.log git-runtime-proof.log + fallback-collection-proof.log git-cli-recovery-proof.log jsonl-driver-proof.log structured-result-proof.log if-no-files-found: warn + - name: Retain failed CLI fixture evidence + if: ${{ failure() && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: snapshot-failed-cli-fixtures-${{ matrix.os }} + path: ${{ runner.temp }}/snapshot-fixtures + include-hidden-files: true + if-no-files-found: warn diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs index 5c3179d3e..7e294d375 100644 --- a/crates/cyber-app/tests/snapshot_reclamation.rs +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -882,3 +882,86 @@ async fn historical_reclamation_reprojects_after_later_fences_and_source_deletio f.app.runtime.shutdown().await; } } + +#[tokio::test] +async fn fallback_collection_honors_disabled_and_zero_retention_at_the_store_writer() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let paths = Paths { + data: root.join("data"), + config: root.join("config"), + state: root.join("state"), + cache: root.join("cache"), + tmp: root.join("tmp"), + }; + paths.ensure().unwrap(); + let project = root.join("project"); + std::fs::create_dir(&project).unwrap(); + std::fs::write(project.join("a.txt"), "unreferenced bytes").unwrap(); + let store = Arc::new( + cyber_store::Store::open(cyber_store::StoreOptions::new( + DatabaseLocation::File(paths.data.join("cyber.db")), + cyber_server::runtime::Runtime::registry(), + )) + .unwrap(), + ); + let config = Arc::new(Mutex::new(json!({}))); + let snapshots = maintenance( + &paths, + config.clone(), + cyber_app::snapshot_reclamation::policy(store), + ); + snapshots + .track(project.to_str().unwrap()) + .await + .unwrap() + .unwrap(); + let namespace = std::fs::read_dir(paths.data.join("snapshot/global")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let retained = age_fallback_collection_images(&namespace); + for settings in [ + json!({"snapshots":false}), + json!({"snapshots":{"retention_days":0}}), + ] { + *config.lock().unwrap() = settings; + snapshots.gc().await; + for (path, bytes) in &retained { + assert_eq!(std::fs::read(path).unwrap(), *bytes); + } + } + *config.lock().unwrap() = json!({}); + snapshots.gc().await; + for (path, _) in retained { + assert!(!path.exists()); + } +} + +fn age_fallback_collection_images(namespace: &std::path::Path) -> Vec<(PathBuf, Vec)> { + let mut retained = vec![]; + for directory in [namespace.join("manifests"), namespace.join("files")] { + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.parent().unwrap().ends_with("manifests") { + let mut manifest: Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + manifest["accessed"] = json!(0); + std::fs::write(&path, manifest.to_string()).unwrap(); + } + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_times( + std::fs::FileTimes::new() + .set_modified(std::time::UNIX_EPOCH + Duration::from_secs(1)), + ) + .unwrap(); + retained.push((path.clone(), std::fs::read(path).unwrap())); + } + } + retained +} diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 41352be46..a4c248ca4 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -141,6 +141,10 @@ impl Env { impl Drop for Env { fn drop(&mut self) { let _ = self.cyber(&["service", "stop"]); + if std::thread::panicking() { + self._dir.disable_cleanup(true); + eprintln!("Preserved failed CLI fixture at {}", self.root.display()); + } } } @@ -3022,3 +3026,19 @@ fn non_git_recovery_inspection_creates_no_namespace_and_starts_no_inference() { assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED); assert!(!env.root.join("cyber-home/data/snapshot/global").exists()); } + +#[test] +fn managed_git_fixture_panic_retains_its_private_evidence() { + let model = scripted_model::ScriptedModel::start(reply); + let env = Env::new(&model); + let root = env.root.clone(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _owned_fixture = env; + panic!("fixture preservation probe"); + })); + assert!(result.is_err()); + assert!(root.join("repo/.git").is_dir()); + // This probe created no Session/native work or unknown evidence. Only its + // own empty fixture is removed; failed recovery fixtures stay retained. + std::fs::remove_dir_all(root).unwrap(); +} diff --git a/crates/cyber-snapshot/src/copy/collection.rs b/crates/cyber-snapshot/src/copy/collection.rs new file mode 100644 index 000000000..945cb40b6 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/collection.rs @@ -0,0 +1,253 @@ +//! Policy-bound fallback collection; unresolved worker/journal evidence stays fenced. +use super::{ENTRY_LIMIT, ImageKind, Manifest, RECORD_LIMIT, now, storage}; +use cap_std::fs::Dir; +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + sync::Arc, +}; +use storage::{Result, fail}; + +/// Prune only unreferenced, old manifests. Journals keep all their objects pinned; +/// an unavailable durable-reference view never authorizes deletion. +pub(crate) fn gc( + data: &Path, + config: &Arc, + protected: &BTreeSet, +) -> Result<()> { + if !data.exists() { + return Ok(()); + } + let data = Dir::open_ambient_dir(data, cap_std::ambient_authority()).map_err(fail)?; + if data.symlink_metadata("snapshot").is_err() { + return Ok(()); + } + let snapshot = storage::existing_child(&data, "snapshot", false)?; + if snapshot.symlink_metadata("global").is_err() { + return Ok(()); + } + let root = storage::existing_child(&snapshot, "global", true)?; + for location in storage::names(&root, 100_000)? { + if !storage::valid_digest(&location) { + return Err("Invalid snapshot namespace".into()); + } + let dir = storage::existing_child(&root, &location, true)?; + let _lock = match storage::lock(&dir, false) { + Ok(lock) => lock, + Err(_) => continue, + }; + gc_location(&dir, &location, config.as_ref(), protected)?; + } + Ok(()) +} +pub(super) fn gc_location( + dir: &Dir, + namespace: &str, + config: &crate::ConfigFn, + protected: &BTreeSet, +) -> Result<()> { + match dir.symlink_metadata("workers") { + Ok(_) => { + let workers = storage::existing_child(dir, "workers", true)?; + if !storage::names(&workers, ENTRY_LIMIT)?.is_empty() { + return Ok(()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(fail(error)), + } + let manifests = storage::existing_child(dir, "manifests", true)?; + let journals = storage::existing_child(dir, "journals", true)?; + // Until an owner resolves/acknowledges native evidence, no journal is evicted. + if !storage::names(&journals, ENTRY_LIMIT)?.is_empty() { + return Ok(()); + } + let objects = storage::existing_child(dir, "files", true)?; + let names = storage::names(&manifests, 100_000)?; + if names.is_empty() && storage::names(&objects, 100_000)?.is_empty() { + return Ok(()); + } + let Some(retention) = Retention::inspect(&manifests, &names, namespace, config)? else { + return Ok(()); + }; + let object_names = object_inventory(&objects)?; + let mut keep = BTreeSet::new(); + for name in names { + if temporary(&name) { + prune_temporary(&manifests, &name, &retention, config)?; + continue; + } + let manifest = read_manifest(&manifests, &name, namespace)?; + let old = now().saturating_sub(manifest.accessed) > retention.seconds(); + if old && !protected.contains(&format!("copy:{name}")) { + retention.verify(config)?; + manifests.remove_file(&name).map_err(fail)?; + } else { + keep.extend( + manifest + .tree + .files + .values() + .map(|entry| entry.object.clone()), + ); + } + } + storage::sync(&manifests)?; + for name in object_names { + if temporary(&name) { + prune_temporary(&objects, &name, &retention, config)?; + continue; + } + if !keep.contains(&name) { + let metadata = storage::open(&objects, Path::new(&name))? + .metadata() + .map_err(fail)?; + let old = metadata + .modified() + .ok() + .and_then(|t| std::time::SystemTime::now().duration_since(t).ok()) + .is_some_and(|age| age.as_secs() > retention.seconds()); + if old { + retention.verify(config)?; + objects.remove_file(&name).map_err(fail)?; + } + } + } + storage::sync(&objects) +} + +/// A complete validated manifest inventory identifies the Location even when its +/// source directory is gone. Never infer a default policy from orphan names. +struct Retention { + location: PathBuf, + settings: serde_json::Value, + days: u64, +} +impl Retention { + fn inspect( + manifests: &Dir, + names: &[String], + namespace: &str, + config: &crate::ConfigFn, + ) -> Result> { + let mut policy: Option = None; + for name in names { + if temporary(name) { + storage::open(manifests, Path::new(name))?; + continue; + } + let manifest = read_manifest(manifests, name, namespace)?; + if let Some(policy) = &policy { + if policy.location != manifest.tree.location { + return Err("Snapshot collection has inconsistent Location evidence".into()); + } + continue; + } + let settings = config(&manifest.tree.location); + let days = settings + .pointer("/snapshots/retention_days") + .and_then(serde_json::Value::as_u64) + .unwrap_or(7); + if settings.get("snapshots") == Some(&serde_json::Value::Bool(false)) || days == 0 { + return Ok(None); + } + policy = Some(Self { + location: manifest.tree.location, + settings, + days, + }); + } + policy + .map(Some) + .ok_or_else(|| "Snapshot collection lacks Location policy evidence".into()) + } + fn seconds(&self) -> u64 { + self.days.saturating_mul(86400) + } + fn verify(&self, config: &crate::ConfigFn) -> Result<()> { + if config(&self.location) != self.settings { + return Err("Snapshot retention policy changed during collection".into()); + } + Ok(()) + } +} +fn object_inventory(objects: &Dir) -> Result> { + let names = storage::names(objects, 100_000)?; + for name in &names { + if !temporary(name) && !storage::valid_digest(name) { + return Err("Unrecognized snapshot object entry".into()); + } + storage::open(objects, Path::new(name))?; + } + Ok(names) +} +fn read_manifest(manifests: &Dir, name: &str, namespace: &str) -> Result { + if !storage::valid_digest(name) { + return Err("Unrecognized snapshot manifest".into()); + } + let manifest: Manifest = + serde_json::from_slice(&storage::read(manifests, name, RECORD_LIMIT)?).map_err(fail)?; + if manifest.tree.version != 1 + || manifest + .tree + .files + .values() + .any(|entry| entry.kind != ImageKind::File) + || storage::digest(&serde_json::to_vec(&manifest.tree).map_err(fail)?) != name + || storage::digest( + format!( + "{}\0{}", + manifest.tree.location.display(), + manifest.tree.root + ) + .as_bytes(), + ) != namespace + { + return Err("Corrupt or foreign snapshot manifest".into()); + } + Ok(manifest) +} + +fn temporary(name: &str) -> bool { + name.strip_prefix("tmp_") + .is_some_and(|id| id.len() == 26 && id.parse::().is_ok()) +} +fn prune_temporary( + dir: &Dir, + name: &str, + retention: &Retention, + config: &crate::ConfigFn, +) -> Result<()> { + let file = storage::open(dir, Path::new(name))?; + let old = file + .metadata() + .map_err(fail)? + .modified() + .ok() + .and_then(|t| std::time::SystemTime::now().duration_since(t).ok()) + .is_some_and(|age| age.as_secs() > retention.seconds()); + if old { + retention.verify(config)?; + dir.remove_file(name).map_err(fail)?; + storage::sync(dir)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_empty_collected_namespace_needs_no_invented_location_policy() { + let temp = tempfile::tempdir().unwrap(); + let dir = Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(); + for name in ["manifests", "journals", "files"] { + storage::private_child(&dir, name).unwrap(); + } + let config = |_: &Path| panic!("empty collection must not invent a Location"); + for _ in 0..2 { + gc_location(&dir, &"a".repeat(64), &config, &BTreeSet::new()).unwrap(); + } + } +} diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index a61e45df3..ce0d19295 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -791,7 +791,15 @@ mod tests { let object = store.object(b"protected preparation bytes").unwrap(); let config: std::sync::Arc = std::sync::Arc::new(|_| serde_json::json!({"snapshots":{"retention_days":0}})); - super::super::gc_location(&store.namespace, &config, &Default::default()).unwrap(); + let namespace = + storage::digest(format!("{}\0{}", store.root.display(), store.root_id).as_bytes()); + super::super::collection::gc_location( + &store.namespace, + &namespace, + config.as_ref(), + &Default::default(), + ) + .unwrap(); assert!(storage::open(&store.objects, Path::new(&object.object)).is_ok()); lifetime.verify().unwrap(); } diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 32c92ae3d..b2f53cee5 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -3,12 +3,14 @@ pub(crate) mod config_inputs; pub(crate) fn object_digest(bytes: &[u8]) -> String { storage::digest(bytes) } +mod collection; pub(crate) mod generated_rules; mod git; pub(crate) mod git_capture; mod git_images; pub(crate) mod git_rules; mod inventory; +pub(crate) use collection::gc; mod links; #[cfg(test)] mod managed_tests; @@ -528,125 +530,6 @@ fn changes(a: &Tree, b: &Tree) -> Vec { .collect() } -/// Prune only unreferenced, old manifests. Journals keep all their objects pinned; -/// an unavailable durable-reference view never authorizes deletion. -pub(crate) fn gc( - data: &Path, - config: &std::sync::Arc, - protected: &BTreeSet, -) -> Result<()> { - if !data.exists() { - return Ok(()); - } - let data = Dir::open_ambient_dir(data, cap_std::ambient_authority()).map_err(fail)?; - if data.symlink_metadata("snapshot").is_err() { - return Ok(()); - } - let snapshot = storage::existing_child(&data, "snapshot", false)?; - if snapshot.symlink_metadata("global").is_err() { - return Ok(()); - } - let root = storage::existing_child(&snapshot, "global", true)?; - for location in storage::names(&root, 100_000)? { - if !storage::valid_digest(&location) { - return Err("Invalid snapshot namespace".into()); - } - let dir = storage::private_child(&root, &location)?; - let _lock = match storage::lock(&dir, false) { - Ok(lock) => lock, - Err(_) => continue, - }; - gc_location(&dir, config, protected)?; - } - Ok(()) -} -fn gc_location( - dir: &Dir, - config: &std::sync::Arc, - protected: &BTreeSet, -) -> Result<()> { - match dir.symlink_metadata("workers") { - Ok(_) => { - let workers = storage::existing_child(dir, "workers", true)?; - if !storage::names(&workers, ENTRY_LIMIT)?.is_empty() { - return Ok(()); - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(fail(error)), - } - let manifests = storage::private_child(dir, "manifests")?; - let journals = storage::private_child(dir, "journals")?; - // Until an owner resolves/acknowledges native evidence, no journal is evicted. - if !storage::names(&journals, ENTRY_LIMIT)?.is_empty() { - return Ok(()); - } - let objects = storage::private_child(dir, "files")?; - let mut keep = BTreeSet::new(); - for name in storage::names(&manifests, 100_000)? { - if temporary(&name) { - prune_temporary(&manifests, &name)?; - continue; - } - if !storage::valid_digest(&name) { - return Err("Unrecognized snapshot manifest".into()); - } - let manifest: Manifest = - serde_json::from_slice(&storage::read(&manifests, &name, RECORD_LIMIT)?) - .map_err(fail)?; - if manifest.tree.version != 1 - || manifest - .tree - .files - .values() - .any(|entry| entry.kind != ImageKind::File) - || storage::digest(&serde_json::to_vec(&manifest.tree).map_err(fail)?) != name - { - return Err("Corrupt snapshot manifest".into()); - } - let days = config(&manifest.tree.location) - .pointer("/snapshots/retention_days") - .and_then(serde_json::Value::as_u64) - .unwrap_or(7); - let old = days != 0 && now().saturating_sub(manifest.accessed) > days.saturating_mul(86400); - if old && !protected.contains(&format!("copy:{name}")) { - manifests.remove_file(&name).map_err(fail)?; - } else { - keep.extend( - manifest - .tree - .files - .values() - .map(|entry| entry.object.clone()), - ); - } - } - storage::sync(&manifests)?; - for name in storage::names(&objects, 100_000)? { - if temporary(&name) { - prune_temporary(&objects, &name)?; - continue; - } - if !storage::valid_digest(&name) { - continue; - } - if !keep.contains(&name) { - let metadata = storage::open(&objects, Path::new(&name))? - .metadata() - .map_err(fail)?; - let old = metadata - .modified() - .ok() - .and_then(|t| std::time::SystemTime::now().duration_since(t).ok()) - .is_some_and(|age| age.as_secs() > 7 * 86400); - if old { - objects.remove_file(&name).map_err(fail)?; - } - } - } - storage::sync(&objects) -} - fn excluded(tree: &Tree, path: &str) -> bool { tree.skipped .iter() @@ -667,23 +550,3 @@ impl Store { Ok(skipped.into_iter().collect()) } } - -fn temporary(name: &str) -> bool { - name.strip_prefix("tmp_") - .is_some_and(|id| id.len() == 26 && id.parse::().is_ok()) -} -fn prune_temporary(dir: &Dir, name: &str) -> Result<()> { - let file = storage::open(dir, Path::new(name))?; - let old = file - .metadata() - .map_err(fail)? - .modified() - .ok() - .and_then(|t| std::time::SystemTime::now().duration_since(t).ok()) - .is_some_and(|age| age.as_secs() > 7 * 86400); - if old { - dir.remove_file(name).map_err(fail)?; - storage::sync(dir)?; - } - Ok(()) -} diff --git a/crates/cyber-snapshot/src/copy/reclamation/paired.rs b/crates/cyber-snapshot/src/copy/reclamation/paired.rs index 96d265849..4c61fa3ad 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/paired.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/paired.rs @@ -204,14 +204,17 @@ fn inventory(journal: &Dir, count: usize) -> Result<(), String> { .map(|index| format!("stage_{index}")) .chain(std::iter::once("intent.json".into())) .collect::>(); - if storage::names(journal, super::super::ENTRY_LIMIT * 3)? - .iter() - .any(|name| !known.contains(name)) + if let Some(name) = storage::names(journal, super::super::ENTRY_LIMIT * 3)? + .into_iter() + .find(|name| !known.contains(name)) { - return Err("Unknown recovery payload or construction entry".into()); + return Err(format!( + "Unknown recovery payload or construction entry: {name:?}" + )); } Ok(()) } + fn planned<'a>(frozen: Option<&'a Plan>, owner: &Owner) -> BTreeMap<&'a str, &'a Payload> { frozen .into_iter() @@ -414,3 +417,26 @@ impl PreparedPair { Ok(Some(pin)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_entry_diagnostic_quotes_the_leaf_and_keeps_its_bytes() { + let temp = tempfile::tempdir().unwrap(); + let dir = Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(); + let name = "foreign\nowner"; + storage::create(&dir, name, b"unknown owner bytes").unwrap(); + let error = inventory(&dir, 0).unwrap_err(); + assert_eq!( + error, + format!("Unknown recovery payload or construction entry: {name:?}") + ); + assert!(!error.contains('\n')); + assert_eq!( + storage::read(&dir, name, 100).unwrap(), + b"unknown owner bytes" + ); + } +} diff --git a/crates/cyber-snapshot/tests/file_copy.rs b/crates/cyber-snapshot/tests/file_copy.rs index 9510d5cfb..ff9800fff 100644 --- a/crates/cyber-snapshot/tests/file_copy.rs +++ b/crates/cyber-snapshot/tests/file_copy.rs @@ -765,3 +765,234 @@ async fn restore_reserves_displaced_preimages_before_changing_source_files() { assert_eq!(std::fs::read(p.root.join("a.txt")).unwrap(), b"new a\n"); assert!(p.snapshots.recovery(&p.dir()).await.unwrap().is_empty()); } + +/// Age both published roots and private orphan/temporary bytes without changing +/// their content identities; collection receives a complete empty reference set. +async fn aged_collection_fixture() -> (Project, Vec<(PathBuf, Vec)>) { + let mut p = Project::new(); + for text in ["first", "second"] { + p.write("a.txt", text); + p.track().await; + } + let namespace = namespace(&p); + let manifests = namespace.join("manifests"); + for entry in std::fs::read_dir(&manifests).unwrap() { + let path = entry.unwrap().path(); + let mut value: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + value["accessed"] = json!(0); + std::fs::write(path, value.to_string()).unwrap(); + } + let orphan = b"unpublished object"; + std::fs::write( + namespace + .join("files") + .join(blake3::hash(orphan).to_hex().as_str()), + orphan, + ) + .unwrap(); + let mut retained = vec![]; + for directory in [manifests, namespace.join("files")] { + std::fs::write( + directory.join(format!("tmp_{}", ulid::Ulid::new())), + b"unpublished temporary", + ) + .unwrap(); + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_times( + std::fs::FileTimes::new() + .set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(1)), + ) + .unwrap(); + retained.push((path.clone(), std::fs::read(path).unwrap())); + } + } + let config = p.config.clone(); + p.snapshots = GitSnapshots::with_policy( + p.data.clone(), + Arc::new(move |_| config.lock().unwrap().clone()), + vec![], + Some(Arc::new(|| Ok(Default::default()))), + ); + (p, retained) +} +fn assert_collection_retained(retained: &[(PathBuf, Vec)]) { + for (path, bytes) in retained { + assert_eq!( + std::fs::read(path).ok().as_ref(), + Some(bytes), + "{}", + path.display() + ); + } +} +#[tokio::test] +async fn collection_disabled_snapshots_preserves_manifests_objects_and_temporary_bytes() { + let (p, retained) = aged_collection_fixture().await; + *p.config.lock().unwrap() = json!({"snapshots":false}); + p.snapshots.gc().await; + assert_collection_retained(&retained); +} +#[tokio::test] +async fn collection_zero_retention_preserves_orphan_and_temporary_bytes() { + let (p, retained) = aged_collection_fixture().await; + *p.config.lock().unwrap() = json!({"snapshots":{"retention_days":0}}); + p.snapshots.gc().await; + assert_collection_retained(&retained); +} +#[tokio::test] +async fn collection_missing_location_evidence_preserves_orphans() { + let (p, mut retained) = aged_collection_fixture().await; + retained.retain(|(path, _)| { + if path.parent().unwrap().ends_with("manifests") + && !path + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("tmp_") + { + std::fs::remove_file(path).unwrap(); + false + } else { + true + } + }); + p.snapshots.gc().await; + assert_collection_retained(&retained); +} +#[tokio::test] +async fn collection_corrupt_inventory_refuses_before_any_manifest_is_removed() { + let (p, retained) = aged_collection_fixture().await; + std::fs::write( + namespace(&p).join("manifests").join("f".repeat(64)), + b"malformed manifest", + ) + .unwrap(); + p.snapshots.gc().await; + assert_collection_retained(&retained); +} +#[tokio::test] +async fn collection_policy_revocation_before_unlink_preserves_the_inventory() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let (mut p, retained) = aged_collection_fixture().await; + let calls = AtomicUsize::new(0); + p.snapshots = GitSnapshots::with_policy( + p.data.clone(), + Arc::new(move |_| { + if calls.fetch_add(1, Ordering::SeqCst) == 0 { + json!({}) + } else { + json!({"snapshots":false}) + } + }), + vec![], + Some(Arc::new(|| Ok(Default::default()))), + ); + p.snapshots.gc().await; + assert_collection_retained(&retained); +} + +#[tokio::test] +async fn collection_orphan_age_uses_the_configured_retention_instead_of_seven_days() { + let (p, _) = aged_collection_fixture().await; + *p.config.lock().unwrap() = json!({"snapshots":{"retention_days":30}}); + let orphan = namespace(&p) + .join("files") + .join(blake3::hash(b"unpublished object").to_hex().as_str()); + std::fs::OpenOptions::new() + .write(true) + .open(&orphan) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified( + std::time::SystemTime::now() - std::time::Duration::from_secs(10 * 86400), + )) + .unwrap(); + p.snapshots.gc().await; + assert_eq!(std::fs::read(orphan).unwrap(), b"unpublished object"); +} + +#[tokio::test] +async fn collection_foreign_location_manifest_does_not_select_another_policy() { + let (p, retained) = aged_collection_fixture().await; + let foreign = Project::new(); + foreign.write("foreign.txt", "another Location"); + let snapshot = foreign.track().await; + let name = snapshot.tree.strip_prefix("copy:").unwrap(); + std::fs::copy( + namespace(&foreign).join("manifests").join(name), + namespace(&p).join("manifests").join(name), + ) + .unwrap(); + p.snapshots.gc().await; + assert_collection_retained(&retained); +} + +#[tokio::test] +async fn collection_policy_revocation_after_manifest_deletion_preserves_objects() { + let (mut p, retained) = aged_collection_fixture().await; + let manifests = namespace(&p).join("manifests"); + for entry in std::fs::read_dir(&manifests).unwrap() { + let path = entry.unwrap().path(); + if path + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("tmp_") + { + std::fs::remove_file(path).unwrap(); + } + } + p.snapshots = GitSnapshots::with_policy( + p.data.clone(), + Arc::new(move |_| { + if std::fs::read_dir(&manifests).unwrap().next().is_none() { + json!({"snapshots":false}) + } else { + json!({}) + } + }), + vec![], + Some(Arc::new(|| Ok(Default::default()))), + ); + p.snapshots.gc().await; + let objects: Vec<_> = retained + .into_iter() + .filter(|(path, _)| path.parent().unwrap().ends_with("files")) + .collect(); + assert_collection_retained(&objects); +} + +#[tokio::test] +async fn collection_temporary_alias_refuses_before_any_manifest_is_removed() { + let (p, retained) = aged_collection_fixture().await; + let target = p.root.join("owner-evidence"); + std::fs::write(&target, b"owner bytes").unwrap(); + let alias = namespace(&p) + .join("manifests") + .join(format!("tmp_{}", ulid::Ulid::new())); + std::os::unix::fs::symlink(&target, &alias).unwrap(); + p.snapshots.gc().await; + assert_collection_retained(&retained); + assert_eq!(std::fs::read(target).unwrap(), b"owner bytes"); + assert!( + std::fs::symlink_metadata(alias) + .unwrap() + .file_type() + .is_symlink() + ); +} +#[tokio::test] +async fn collection_unknown_object_refuses_before_any_manifest_is_removed() { + let (p, retained) = aged_collection_fixture().await; + let unknown = namespace(&p).join("files").join("foreign-owner-evidence"); + std::fs::write(&unknown, b"unknown owner").unwrap(); + p.snapshots.gc().await; + assert_collection_retained(&retained); + assert_eq!(std::fs::read(unknown).unwrap(), b"unknown owner"); +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 2554b36d9..72e2ffacf 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -65,6 +65,18 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. returning aggregate exit 1 and an `end` error. Interrupted/protocol exit codes retain precedence. This does not complete the ownership/backpressure findings. +5. **Fixed locally: fallback collection ignored disabled retention policy.** + Complete manifest/entry preflight now binds the namespace Location, protects + disabled/zero-retention data, honors configured orphan aging and rechecks policy + before each unlink. Final fallback/App integration coverage passes 44 cases; + durable collection/retirement evidence and full metadata/CAS cleanup remain open. + Native run 42728260672 at 1be03642 passes both Linux jobs but fails managed Git + cleanup-plan preparation on macOS ARM because of an unidentified unknown entry. + Keep that fence: the diagnostic now quotes the leaf, failed CLI fixture roots + are retained/uploaded, and independent structured/JSONL checks no longer depend + on earlier recovery success. This is an open construction/acceptance failure, + not a timeout or an accepted native result. + Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; formatting and whitespace diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index f9d1759c6..cd97d379d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -159,6 +159,15 @@ finish acknowledged payload reclamation but must not delete unacknowledged evide or replay filesystem resolution. Unknown files, aliases and identity changes refuse cleanup visibly. No direct user removal of private journal directories is required. +Fallback manifest/object collection SHALL establish the Location's current +retention policy from a complete valid, namespace-bound manifest inventory before +deletion. Disabled snapshots and zero retention preserve manifests, orphan objects +and temporary files. Configured retention applies to orphan/temporary aging too; +policy changes refuse before the next unlink. Corrupt/foreign manifests, unknown +object entries and temporary aliases refuse before manifest deletion. An empty +namespace is a no-op. Orphans without surviving Location evidence remain retained; +durable collection/retirement evidence for those namespaces remains part of M6. + ## Managed checkout payload reclamation Reclamation of acknowledged regular-file and symlink Git journal payloads SHALL reacquire diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 6b306ee50..2d4c807db 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -497,3 +497,10 @@ - **THEN** it SHALL pin and unlink the journal link inode without opening its target, including dangling targets - **AND** an equal-target replacement inode or changed metadata SHALL refuse before deletion; a missing leaf SHALL require the exact previously committed plan - **AND** later owner source links, consuming receipts and native ownership metadata SHALL remain intact + +#### Scenario: Fallback collection has disabled or unavailable retention policy +- **WHEN** fallback manifest, orphan-object or temporary-file collection begins +- **THEN** a complete valid manifest inventory SHALL establish the namespace-bound Location policy before deletion; absent, corrupt or foreign Location evidence SHALL refuse collection +- **AND** unknown object entries and unsupported temporary-file aliases SHALL refuse before any manifest unlink; an already empty namespace SHALL remain a no-op +- **AND** disabled snapshots or zero retention SHALL preserve those bytes, and orphan/temporary aging SHALL use the configured retention duration +- **AND** a changed policy SHALL refuse before the next unlink, preserving remaining manifests and objects even if earlier eligible manifests were already removed diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 15c5f95f7..98fc17bf2 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -886,3 +886,50 @@ macOS ARM/Intel. It predates steering and this cancellation extension. Run its Linux x64 job has passed, with the remaining jobs pending at this checkpoint. Neither run accepts the new cancelled-Git cleanup source. Queue its own native proof and retain each run's actual tested source identity. + + +Fallback collection retention continuation (2026-11-09): six new regressions +initially fail because disabled capture still prunes manifests, zero retention +still prunes temporary/orphan bytes, orphan aging is fixed at seven days, corrupt +or missing Location evidence is discovered after deletion, and policy revocation +is not rechecked. Collection now preflights the complete namespace-bound manifest +inventory and known regular object/temporary entries before manifest unlinks, +retains disabled/zero-retention bytes, applies configured retention to orphan and +temporary aging, and refuses a changed configuration before each unlink. Foreign +Location manifests, temporary aliases and unknown object entries refuse before +any manifest deletion; already empty namespaces are no-ops. The native collector +is isolated in copy/collection.rs and still uses the existing serialized Store +reference callback. No native worker/journal fence is relaxed and no API, SDK or +receipt schema changes. Missing surviving Location evidence protects orphan bytes; +durable collection/retirement evidence, complete metadata/CAS reclamation and +partial construction remain required by M6 rather than being inferred from names. + +Final local verification: 28 fallback integration tests and 16 App reclamation +cases pass, including a real Store-writer disabled/zero-policy/reenabled-collection +case and existing reference-writer exclusion, cap-pressure, unknown-owner, native +process-death, restart and receipt regressions. The additional empty-namespace +unit and quoted unknown-entry diagnostic unit pass. A focused App unknown-payload +rerun passes after the diagnostic change, and the new CLI panic-preservation probe +passes. Earlier in this turn, before the final preflight refinements, the complete +snapshot suite passed 188 tests with seven intentional subprocess/diagnostic skips; +that earlier run is not claimed as final-source full-suite acceptance. Final +all-target snapshot/App/CLI Clippy and formatting/whitespace pass. Strict OpenSpec +passes 100 items; spec lint reports zero errors and 25 warnings. Focused code and +security review found no additional issue in these changes after the preflight +fixes; the macOS construction failure and complete branch review remain open. Native platform, SDK/generated +artifact and full-workspace delivery acceptance remain open. + +Source-bound native failure: run 42728260672 at 1be03642 passes Linux x64/ARM, +but macOS ARM fails the managed Git process-death recovery case at cleanup-plan +preparation with "Unknown recovery payload or construction entry". Its adapter, +core marker and App Git acceptance steps passed; JSONL and structured steps were +skipped after the failure. The log does not identify the entry, so its type and +cause are not established and the fence must not be bypassed. Paired cleanup now +quotes the first bounded unexpected leaf in its refusal, with a unit case proving +newlines are escaped and bytes remain intact. CLI fixtures now retain their own +private roots on panic after service stop, with a catch-unwind regression; native +CI uploads those failed scripted-fixture directories including hidden Git metadata. +Independent JSONL/structured checks run unless cancelled even after an earlier +recovery failure, without relaxing the overall job gate. The native workflow also +selects the complete App reclamation suite. These edits provide diagnostics and +coverage, not a fix or passing acceptance for the macOS construction failure. From 2ec1b1c5412fb62ebbb20dbb1ab593a536257517 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 00:06:21 -0400 Subject: [PATCH 60/87] Bind prepared intent updates to preallocated native slots --- .github/workflows/non-git-snapshot-proof.yml | 10 + .../cyber-app/tests/snapshot_git_runtime.rs | 6 +- .../cyber-app/tests/snapshot_reclamation.rs | 75 ++++ crates/cyber-cli/tests/branches_cli.rs | 45 +- crates/cyber-cli/tests/exec_output_schema.rs | 53 ++- .../cyber-snapshot/src/copy/intent_slots.rs | 415 ++++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 1 + crates/cyber-snapshot/src/copy/ownership.rs | 2 +- crates/cyber-snapshot/src/copy/reclamation.rs | 21 + .../src/copy/reclamation/cancelled.rs | 2 + .../src/copy/reclamation/paired.rs | 40 +- crates/cyber-snapshot/src/copy/resolution.rs | 17 +- crates/cyber-snapshot/src/copy/restore.rs | 73 ++- docs/goal-delivery.md | 22 +- .../managed-recovery.md | 21 + .../specs/snapshots-checkpoints/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 35 ++ 17 files changed, 797 insertions(+), 48 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/intent_slots.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 01b40b4ad..b643ff40b 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -91,6 +91,15 @@ jobs: set -euo pipefail mkdir -p "$TMPDIR" cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_git_ -- --nocapture --test-threads=1 2>&1 | tee git-cli-recovery-proof.log + - name: Pre-effect preparation and TUI lost-reply reconciliation + if: ${{ !cancelled() }} + env: + TMPDIR: ${{ runner.temp }}/snapshot-fixtures + shell: bash + run: | + set -euo pipefail + mkdir -p "$TMPDIR" + cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli preparation_ -- --nocapture --test-threads=1 2>&1 | tee preparation-client-proof.log - name: JSONL driver and atomic last-message regressions if: ${{ !cancelled() }} shell: bash @@ -116,6 +125,7 @@ jobs: git-runtime-proof.log fallback-collection-proof.log git-cli-recovery-proof.log + preparation-client-proof.log jsonl-driver-proof.log structured-result-proof.log if-no-files-found: warn diff --git a/crates/cyber-app/tests/snapshot_git_runtime.rs b/crates/cyber-app/tests/snapshot_git_runtime.rs index 6794f7b41..2834c82ad 100644 --- a/crates/cyber-app/tests/snapshot_git_runtime.rs +++ b/crates/cyber-app/tests/snapshot_git_runtime.rs @@ -348,7 +348,11 @@ async fn reclaim_restore_payloads(f: &Fixture, receipt: &SnapshotRestoreReceipt) 0 ); assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); - assert_eq!(std::fs::read_dir(journal).unwrap().count(), 1); + let header: Value = serde_json::from_slice(&intent).unwrap(); + assert_eq!(header["version"], 7); + assert!(header["updates"].is_object()); + assert!(journal.join("intent.next").is_file()); + assert_eq!(std::fs::read_dir(journal).unwrap().count(), 2); assert_reclaimed_source_and_receipt(f, receipt, inference).await; } diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs index 7e294d375..51b843771 100644 --- a/crates/cyber-app/tests/snapshot_reclamation.rs +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -364,6 +364,7 @@ struct Cut { inner: Arc, after_delete: bool, pause: Option, + tamper: Option, } impl Policy for Cut { fn receipt( @@ -378,9 +379,14 @@ impl Policy for Cut { fn reclaim(&self, plan: records::Plan, delete: Delete) -> Result { let after = self.after_delete; let pause = self.pause.clone(); + let tamper = self.tamper.clone(); self.inner.reclaim( plan, Box::new(move || { + if let Some(path) = tamper { + std::fs::write(path, b"later metadata owner bytes").unwrap(); + return delete(); + } if after { delete()?; } @@ -409,6 +415,7 @@ async fn durable_plan_survives_interruption_before_and_after_private_deletion() let policy = Arc::new(Cut { inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), after_delete, + tamper: None, pause: None, }); let cut = maintenance(&f.paths, f.config.clone(), policy); @@ -444,6 +451,7 @@ async fn reclamation_process_worker() { let policy = Arc::new(Cut { inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), after_delete: std::env::var("CYBER_RECLAMATION_AFTER_DELETE").unwrap() == "true", + tamper: None, pause: Some(control.join("ready")), }); let cut = maintenance(&f.paths, f.config.clone(), policy); @@ -730,6 +738,7 @@ async fn planned_without_deletion(f: &Fixture) { let policy = Arc::new(Cut { inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), after_delete: false, + tamper: None, pause: None, }); assert!( @@ -965,3 +974,69 @@ fn age_fallback_collection_images(namespace: &std::path::Path) -> Vec<(PathBuf, } retained } + +#[tokio::test] +async fn a_foreign_intent_spare_refuses_cleanup_before_plan_without_touching_source() { + use std::os::unix::fs::PermissionsExt; + let f = Fixture::new().await; + f.age(); + let active = std::fs::read(f.journal.join("intent.json")).unwrap(); + let spare = f.journal.join("intent.next"); + let bytes = std::fs::read(&spare).unwrap(); + let retained = f.paths.tmp.join("retained-original-spare"); + std::fs::rename(&spare, &retained).unwrap(); + std::fs::write(&spare, &bytes).unwrap(); + std::fs::set_permissions(&spare, std::fs::Permissions::from_mode(0o600)).unwrap(); + let inference = f.model.requests().len(); + let error = f.reclaim().await.unwrap_err(); + assert!(error.contains("Intent slot identity changed"), "{error}"); + assert!(f.record().is_none()); + assert!(f.journal.join("stage_0").exists()); + assert_eq!(std::fs::read(&spare).unwrap(), bytes); + assert_eq!( + std::fs::read(f.journal.join("intent.json")).unwrap(), + active + ); + assert_eq!( + std::fs::read_to_string(f.project.join("file_0")).unwrap(), + "before\n".repeat(4096) + ); + std::fs::remove_file(&spare).unwrap(); + std::fs::rename(retained, &spare).unwrap(); + assert!(f.reclaim().await.unwrap() > 0); + assert_eq!(std::fs::read(spare).unwrap(), bytes); + assert_eq!(f.model.requests().len(), inference); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn a_spare_metadata_change_after_planning_refuses_before_private_unlinks() { + let f = Fixture::new().await; + f.age(); + let spare = f.journal.join("intent.next"); + let original = std::fs::read(&spare).unwrap(); + let cut = maintenance( + &f.paths, + f.config.clone(), + Arc::new(Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after_delete: false, + pause: None, + tamper: Some(spare.clone()), + }), + ); + let error = cut.reclaim(f.project.to_str().unwrap()).await.unwrap_err(); + assert!( + error.contains("metadata inode or generation changed"), + "{error}" + ); + assert!(!f.record().unwrap().completed); + assert!(f.journal.join("stage_0").exists() && f.journal.join("stage_1").exists()); + assert_eq!( + std::fs::read(&spare).unwrap(), + b"later metadata owner bytes" + ); + std::fs::write(&spare, original).unwrap(); + assert!(f.reclaim().await.unwrap() > 0); + f.app.runtime.shutdown().await; +} diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index a4c248ca4..bc68c6c83 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1420,7 +1420,7 @@ fn assert_prepared_payload_cleanup(env: &Env, reply: &Value) { std::fs::read(journal.join("preparation.json")).unwrap(), cancellation ); - assert_eq!(std::fs::read_dir(&journal).unwrap().count(), 2); + assert_retained_intent_slots(&journal, &intent, 1); assert!( std::fs::symlink_metadata(source.join("bulk_0000")) .unwrap() @@ -1465,6 +1465,19 @@ fn assert_prepared_managed_guards( assert_managed_cleanup_refusals(env, maintenance, &receipt, &bindings[0], &owner); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_retained_intent_slots(journal: &std::path::Path, intent: &[u8], extra_metadata: usize) { + assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); + let header: Value = serde_json::from_slice(intent).unwrap(); + assert_eq!(header["version"], 7); + assert!(header["updates"].is_object()); + assert!(journal.join("intent.next").is_file()); + assert_eq!( + std::fs::read_dir(journal).unwrap().count(), + 2 + extra_metadata + ); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_prepared_interrupted_plan( maintenance: &reclamation_support::Maintenance, @@ -2240,12 +2253,7 @@ fn assert_paired_payload_cleanup(env: &Env, result: &Value) { records::Plan::id(&receipt).unwrap() ); for (journal, intent) in journals.iter().zip(intents) { - assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); - assert_eq!( - std::fs::read_dir(journal).unwrap().count(), - 1, - "only the acknowledged intent remains" - ); + assert_retained_intent_slots(journal, &intent, 0); } assert_eq!( runtime @@ -2855,14 +2863,7 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { .unwrap(), ) .unwrap(); - assert_eq!(intent["version"], 3); - assert_eq!(intent["ownership"]["version"], 1); - assert_eq!( - intent["ownership"]["files"].as_array().unwrap().len(), - intent["changes"].as_array().unwrap().len() - ); - assert_eq!(intent["completed"], true); - assert_eq!(intent["acknowledged"], true); + assert_copy_restore_intent(&intent); let origin = &intent["origin"]; assert_eq!(origin["session_id"], session); assert_eq!(origin["binding"]["journal_id"], journal); @@ -2904,6 +2905,20 @@ fn assert_owned_snapshot_receipt(env: &Env, session: &str, journal: &str) { assert_eq!(origin["session_created_id"], created); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_copy_restore_intent(intent: &Value) { + assert_eq!(intent["version"], 7); + assert_eq!(intent["updates"]["version"], 1); + assert_eq!(intent["updates"]["format"], 3); + assert_eq!(intent["ownership"]["version"], 1); + assert_eq!( + intent["ownership"]["files"].as_array().unwrap().len(), + intent["changes"].as_array().unwrap().len() + ); + assert_eq!(intent["completed"], true); + assert_eq!(intent["acknowledged"], true); +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_restore_receipt_binding(activity: &Value, origin: &Value) { let receipt = &activity["snapshot_restore_receipt"]; diff --git a/crates/cyber-cli/tests/exec_output_schema.rs b/crates/cyber-cli/tests/exec_output_schema.rs index c16a57d64..65f8e7ea7 100644 --- a/crates/cyber-cli/tests/exec_output_schema.rs +++ b/crates/cyber-cli/tests/exec_output_schema.rs @@ -201,7 +201,6 @@ fn collect(mut child: Child) -> Output { panic!("structured exec did not settle after cancellation: {output:?}"); } -#[cfg(unix)] fn wait_for_turns(provider: &Provider, child: &mut Child, count: usize) { for _ in 0..2000 { if provider.turns() >= count { @@ -219,7 +218,7 @@ fn wait_for_turns(provider: &Provider, child: &mut Child, count: usize) { } #[test] -fn budgets_and_timeouts_keep_exit_4_when_the_open_structured_attempt_is_interrupted() { +fn budgets_and_short_timeouts_preserve_exit_4_before_or_during_inference() { for (stopping, reason) in [ (["--max-tokens", "1"], "budget_exceeded"), (["--timeout", "2s"], "timeout"), @@ -237,16 +236,52 @@ fn budgets_and_timeouts_keep_exit_4_when_the_open_structured_attempt_is_interrup .spawn() .unwrap(), ); - assert_eq!(output.status.code(), Some(4), "{output:?}"); - let result: Value = serde_json::from_str(stdout(&output).trim()).unwrap(); - assert_eq!(result["exit_code"], 4, "{result}"); - assert_eq!(result["stop_reason"], reason, "{result}"); - assert_eq!(result["result"], Value::Null); - assert!(result["error"].as_str().is_some(), "{result}"); - assert!((1..=2).contains(&provider.turns())); + assert_stopped_schema(&output, reason); + // Auxiliary usage or the wall deadline may stop an admitted schema + // before its first main turn. Neither requires inference after the stop. + assert!( + provider.turns() <= 2, + "{reason}: {} main turns; {output:?}", + provider.turns() + ); } } +fn assert_stopped_schema(output: &Output, reason: &str) { + assert_eq!(output.status.code(), Some(4), "{output:?}"); + let result: Value = serde_json::from_str(stdout(output).trim()).unwrap(); + assert_eq!(result["exit_code"], 4, "{result}"); + assert_eq!(result["stop_reason"], reason, "{result}"); + assert_eq!(result["result"], Value::Null); + assert!(result["error"].as_str().is_some(), "{result}"); +} + +#[test] +fn a_wall_timeout_during_a_verified_structured_correction_keeps_exit_4() { + let provider = Provider::start(&[r#"{"wrong":1}"#, "wait"]); + let env = Env::new(&provider); + let mut child = env + .command(&[ + "--output-schema", + &env.schema(), + "--format", + "json", + "--timeout", + "10s", + "audit", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + // This separate case proves the in-flight correction precondition instead + // of assuming startup completed before a short wall deadline. + wait_for_turns(&provider, &mut child, 2); + let output = collect(child); + assert_stopped_schema(&output, "timeout"); + assert_eq!(provider.turns(), 2); +} + #[cfg(unix)] #[test] fn sigint_during_a_structured_correction_keeps_exit_130() { diff --git a/crates/cyber-snapshot/src/copy/intent_slots.rs b/crates/cyber-snapshot/src/copy/intent_slots.rs new file mode 100644 index 000000000..6a919c227 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/intent_slots.rs @@ -0,0 +1,415 @@ +//! Two preallocated intent inodes; torn spare bytes never replace the active intent. +use super::{restore::Journal, storage}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions, OpenOptionsExt}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::File, + io::{Seek, SeekFrom, Write}, + path::Path, +}; +use storage::{Result, fail}; + +pub(super) const ACTIVE: &str = "intent.json"; +pub(super) const SPARE: &str = "intent.next"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Binding { + version: u32, + format: u32, + first: String, + second: String, +} +impl Binding { + pub fn budget(format: u32) -> Self { + Self { + version: 1, + format, + first: "0".repeat(storage::IDENTITY_MAX_BYTES), + second: "1".repeat(storage::IDENTITY_MAX_BYTES), + } + } + pub fn format(&self) -> Result { + if self.version != 1 + || !matches!(self.format, 3 | 5 | 6) + || !super::ownership::valid_identity(&self.first) + || !super::ownership::valid_identity(&self.second) + || self.first == self.second + { + return Err("Invalid intent slot binding".into()); + } + Ok(self.format) + } + fn matches(&self, first: &File, second: &File) -> Result<()> { + self.format()?; + storage::validate_lock(first)?; + storage::validate_lock(second)?; + let (a, b) = (storage::identity(first)?, storage::identity(second)?); + if !((a == self.first && b == self.second) || (a == self.second && b == self.first)) { + return Err("Intent slot identity changed".into()); + } + Ok(()) + } + pub fn verify(&self, dir: &Dir) -> Result<()> { + self.matches( + &storage::open(dir, Path::new(ACTIVE))?, + &storage::open(dir, Path::new(SPARE))?, + ) + } +} + +/// Retain the actual creating descriptors until the prepared binding is published. +/// A cut before publication remains non-admitted construction evidence. +pub(super) struct Construction { + active: File, + spare: File, + active_generation: String, + spare_generation: String, +} +impl Construction { + pub fn create(dir: &Dir, journal: &Journal) -> Result { + // Keep the initial intent absent until its complete bytes are durable, + // preserving pre-admission cancellation semantics for a torn creation. + let temporary = format!("tmp_{}", ulid::Ulid::new()); + let active = storage::create(dir, &temporary, &encoded(journal)?)?; + rustix::fs::renameat_with( + dir, + &temporary, + dir, + ACTIVE, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(fail)?; + storage::sync(dir)?; + let spare = storage::create(dir, SPARE, b"")?; + storage::sync(dir)?; + Ok(Self { + active_generation: super::ownership::generation(&active)?, + spare_generation: super::ownership::generation(&spare)?, + active, + spare, + }) + } + pub fn binding(&self, dir: &Dir, format: u32) -> Result { + if super::ownership::generation(&self.active)? != self.active_generation + || super::ownership::generation(&self.spare)? != self.spare_generation + { + return Err("Intent construction metadata changed".into()); + } + let binding = Binding { + version: 1, + format, + first: storage::identity(&self.active)?, + second: storage::identity(&self.spare)?, + }; + binding.verify(dir)?; + Ok(binding) + } + pub fn publish(&self, dir: &Dir, journal: &Journal) -> Result<()> { + let expected = self.binding(dir, journal.format_version()?)?; + if journal.updates.as_ref() != Some(&expected) { + return Err("Prepared intent differs from its creating descriptors".into()); + } + write_bound(dir, journal, true, |_| Ok(())) + } +} +fn encoded(journal: &Journal) -> Result> { + let bytes = serde_json::to_vec(journal).map_err(fail)?; + if bytes.len() as u64 > super::RECORD_LIMIT { + return Err("Restore journal exceeds limit".into()); + } + Ok(bytes) +} +fn writable(dir: &Dir, name: &str) -> Result { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .follow(FollowSymlinks::No) + .custom_flags(libc::O_NONBLOCK); + let file = dir.open_with(name, &options).map_err(fail)?.into_std(); + storage::validate_lock(&file)?; + Ok(file) +} +pub(super) fn write(dir: &Dir, journal: &Journal) -> Result<()> { + write_before_exchange(dir, journal, |_| Ok(())) +} +fn write_before_exchange( + dir: &Dir, + journal: &Journal, + before_exchange: impl FnOnce(&mut File) -> Result<()>, +) -> Result<()> { + write_bound(dir, journal, false, before_exchange) +} +fn write_bound( + dir: &Dir, + journal: &Journal, + initializing: bool, + before_exchange: impl FnOnce(&mut File) -> Result<()>, +) -> Result<()> { + journal.format_version()?; + let binding = journal + .updates + .as_ref() + .ok_or("Missing intent slot binding")?; + let active = writable(dir, ACTIVE)?; + let mut spare = writable(dir, SPARE)?; + binding.matches(&active, &spare)?; + if !initializing { + let current: Journal = + serde_json::from_slice(&storage::read(dir, ACTIVE, super::RECORD_LIMIT)?) + .map_err(fail)?; + current.format_version()?; + if current.version != 7 + || current.updates.as_ref() != Some(binding) + || current.origin != journal.origin + { + return Err("Active intent no longer owns these update slots".into()); + } + } + let bytes = encoded(journal)?; + spare.seek(SeekFrom::Start(0)).map_err(fail)?; + spare.set_len(0).map_err(fail)?; + spare.write_all(&bytes).map_err(fail)?; + before_exchange(&mut spare)?; + storage::sync_file(&spare)?; + binding.verify(dir)?; + rustix::fs::renameat_with(dir, ACTIVE, dir, SPARE, rustix::fs::RenameFlags::EXCHANGE) + .map_err(fail)?; + storage::sync(dir)?; + binding.verify(dir) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::copy::{Options, Store, now}; + use cyber_server::runtime::{SnapshotBinding, SnapshotOrigin}; + const ID: &str = "restore_slot_fixture"; + fn options() -> Options { + Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + } + } + fn fixture() -> (tempfile::TempDir, Store, Dir, Journal) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("owner"), b"owner bytes").unwrap(); + let store = + Store::open(&temp.path().join("data"), root.to_str().unwrap(), options()).unwrap(); + let dir = storage::private_child(&store.journals, ID).unwrap(); + let mut journal = Journal { + version: 3, + updates: None, + backend: None, + location: store.root.clone(), + root: store.root_id.clone(), + created: now(), + completed: false, + changes: vec![], + applied: vec![], + acknowledged: false, + settled_by: None, + ownership: None, + context: None, + resolution: None, + origin: Some(SnapshotOrigin { + version: 1, + activity_id: "op_slots".into(), + session_id: "ses_slots".into(), + session_created_id: "evt_slots".into(), + worktree_id: None, + binding: SnapshotBinding { + directory: store.root.clone(), + root_identity: store.root_id.clone(), + namespace_identity: storage::dir_identity(&store.journals).unwrap(), + journal_id: ID.into(), + journal_identity: storage::dir_identity(&dir).unwrap(), + lock_identity: Some(store.lock_identity().unwrap()), + }, + }), + }; + let construction = Construction::create(&dir, &journal).unwrap(); + journal.ownership = + Some(crate::copy::ownership::Ownership::prepare(&store, &dir, &[]).unwrap()); + journal.updates = Some(construction.binding(&dir, journal.version).unwrap()); + journal.version = 7; + construction.publish(&dir, &journal).unwrap(); + (temp, store, dir, journal) + } + #[test] + fn repeated_updates_keep_two_bound_inodes_and_the_immutable_digest() { + let (_temp, store, dir, mut journal) = fixture(); + let before = store.worker_digest(journal.origin.as_ref()).unwrap(); + let binding = journal.updates.clone(); + for _ in 0..8 { + journal.completed = !journal.completed; + Store::record(&dir, &journal).unwrap(); + let (_, reopened) = store.journal(ID).unwrap(); + assert_eq!(reopened.completed, journal.completed); + assert_eq!(reopened.updates, binding); + assert_eq!( + store.worker_digest(journal.origin.as_ref()).unwrap(), + before + ); + assert_eq!(storage::names(&dir, 10).unwrap(), [ACTIVE, SPARE]); + } + } + #[test] + fn slot_native_identities_are_immutable_receipt_input() { + let (_temp, _store, _dir, journal) = fixture(); + let digest = crate::copy::resolution::immutable_digest(&journal).unwrap(); + let mut changed = journal.clone(); + changed.updates.as_mut().unwrap().first = "1:2:3:4".into(); + assert_ne!( + crate::copy::resolution::immutable_digest(&changed).unwrap(), + digest + ); + } + #[test] + fn equal_byte_foreign_slot_missing_route_and_alias_refuse_without_writes() { + let (temp, _store, dir, journal) = fixture(); + let original = storage::read(&dir, ACTIVE, super::super::RECORD_LIMIT).unwrap(); + let spare = storage::read(&dir, SPARE, super::super::RECORD_LIMIT).unwrap(); + let path = temp.path().join("retained-spare"); + dir.rename( + SPARE, + &Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(), + "retained-spare", + ) + .unwrap(); + assert!(write(&dir, &journal).is_err()); + storage::create(&dir, SPARE, &spare).unwrap(); + assert!( + write(&dir, &journal) + .unwrap_err() + .contains("identity changed") + ); + assert_eq!( + storage::read(&dir, SPARE, super::super::RECORD_LIMIT).unwrap(), + spare + ); + dir.remove_file(SPARE).unwrap(); + rustix::fs::symlinkat(&path, &dir, SPARE).unwrap(); + assert!(write(&dir, &journal).is_err()); + assert_eq!( + storage::read(&dir, ACTIVE, super::super::RECORD_LIMIT).unwrap(), + original + ); + assert_eq!(std::fs::read(path).unwrap(), spare); + } + #[test] + fn legacy_records_cannot_claim_a_spare_by_name_or_partial_binding() { + let (_temp, store, dir, mut journal) = fixture(); + journal.version = 3; + let before = storage::read(&dir, ACTIVE, super::super::RECORD_LIMIT).unwrap(); + assert!(Store::record(&dir, &journal).is_err()); + assert_eq!( + storage::read(&dir, ACTIVE, super::super::RECORD_LIMIT).unwrap(), + before + ); + let bytes = serde_json::to_vec(&journal).unwrap(); + // Test corruption only: overwrite the current inode instead of invoking + // the guarded production writer with an invalid version/binding pair. + let mut active = writable(&dir, ACTIVE).unwrap(); + active.set_len(0).unwrap(); + active.write_all(&bytes).unwrap(); + storage::sync_file(&active).unwrap(); + assert!(store.journal(ID).is_err()); + journal.version = 7; + journal.updates = None; + assert!(Store::record(&dir, &journal).is_err()); + assert_eq!(storage::names(&dir, 10).unwrap(), [ACTIVE, SPARE]); + active.seek(SeekFrom::Start(0)).unwrap(); + active.set_len(0).unwrap(); + active + .write_all(&serde_json::to_vec(&journal).unwrap()) + .unwrap(); + storage::sync_file(&active).unwrap(); + assert!(store.journal(ID).is_err()); + } + #[test] + #[ignore = "subprocess fixture invoked by torn-spare process proof"] + fn intent_update_child() { + let root = std::path::PathBuf::from(std::env::var_os("CYBER_INTENT_SLOT_CHILD").unwrap()); + let store = Store::recover_existing( + &root.join("data"), + root.join("work").to_str().unwrap(), + options(), + ) + .unwrap(); + let (dir, mut journal) = store.journal(ID).unwrap(); + journal.completed = true; + let _ = write_before_exchange(&dir, &journal, |spare| { + spare.seek(SeekFrom::Start(0)).map_err(fail)?; + spare.set_len(1).map_err(fail)?; + spare.write_all(b"{").map_err(fail)?; + storage::sync_file(spare)?; + std::fs::write(root.join("ready"), b"partial spare, active unchanged").map_err(fail)?; + loop { + std::thread::sleep(std::time::Duration::from_secs(1)); + } + }); + panic!("child passed the cut"); + } + #[test] + fn process_death_during_spare_write_preserves_active_intent_and_reopens_without_replay() { + use std::os::unix::process::ExitStatusExt; + let (temp, store, dir, journal) = fixture(); + let expected = storage::read(&dir, ACTIVE, super::super::RECORD_LIMIT).unwrap(); + let binding = journal.updates.clone(); + drop(dir); + drop(store); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "copy::intent_slots::tests::intent_update_child", + "--ignored", + "--nocapture", + ]) + .env("CYBER_INTENT_SLOT_CHILD", temp.path()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + let ready = loop { + if temp.path().join("ready").exists() { + break true; + } + if child.try_wait().unwrap().is_some() || std::time::Instant::now() >= deadline { + break false; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + }; + let _ = child.kill(); + let status = child.wait().unwrap(); + assert!(ready, "child did not reach the cut: {status}"); + assert_eq!(status.signal(), Some(9)); + let store = Store::recover_existing( + &temp.path().join("data"), + temp.path().join("work").to_str().unwrap(), + options(), + ) + .unwrap(); + let (dir, mut reopened) = store.journal(ID).unwrap(); + assert!(!reopened.completed); + assert_eq!( + storage::read(&dir, ACTIVE, super::super::RECORD_LIMIT).unwrap(), + expected + ); + assert_eq!(storage::read(&dir, SPARE, 100).unwrap(), b"{"); + reopened.completed = true; + Store::record(&dir, &reopened).unwrap(); + assert!(store.journal(ID).unwrap().1.completed); + assert_eq!(store.journal(ID).unwrap().1.updates, binding); + assert_eq!( + std::fs::read(temp.path().join("work/owner")).unwrap(), + b"owner bytes" + ); + assert_eq!(storage::names(&dir, 10).unwrap(), [ACTIVE, SPARE]); + } +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index b2f53cee5..1ace9a7ec 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -9,6 +9,7 @@ mod git; pub(crate) mod git_capture; mod git_images; pub(crate) mod git_rules; +mod intent_slots; mod inventory; pub(crate) use collection::gc; mod links; diff --git a/crates/cyber-snapshot/src/copy/ownership.rs b/crates/cyber-snapshot/src/copy/ownership.rs index c4c5cb02a..bd53fc65f 100644 --- a/crates/cyber-snapshot/src/copy/ownership.rs +++ b/crates/cyber-snapshot/src/copy/ownership.rs @@ -260,7 +260,7 @@ fn parent_paths(changes: &[Change]) -> Result> { Ok(paths) } -fn valid_identity(value: &str) -> bool { +pub(super) fn valid_identity(value: &str) -> bool { if value.is_empty() || value.len() > storage::IDENTITY_MAX_BYTES { return false; } diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index dcc3367b7..9c53713f2 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -21,6 +21,7 @@ struct Prepared { store: Store, journal: Dir, intent: MetadataPin, + spare: Option, cancellation: Option, phase: MetadataPin, workers: Dir, @@ -46,6 +47,16 @@ impl MetadataPin { file, }) } + fn spare(dir: &Dir, record: &Journal) -> Result, String> { + record + .updates + .as_ref() + .map(|binding| { + binding.verify(dir)?; + Self::open(dir, super::intent_slots::SPARE) + }) + .transpose() + } fn verify(&self, dir: &Dir, name: &str) -> Result<(), String> { let route = storage::open(dir, Path::new(name))?; storage::validate_lock(&route)?; @@ -226,6 +237,10 @@ fn prepare( .collect::>() }); let mut known = BTreeSet::from(["intent.json".to_owned()]); + if let Some(binding) = &record.updates { + binding.verify(&journal)?; + known.insert(super::intent_slots::SPARE.to_owned()); + } for (index, change) in record.changes.iter().enumerate() { let name = format!("stage_{index}"); known.insert(name.clone()); @@ -281,6 +296,7 @@ fn prepare( return Err("Reclamation plan differs from retained tombstone".into()); } let intent = MetadataPin::open(&journal, "intent.json")?; + let spare = MetadataPin::spare(&journal, &record)?; let workers = storage::existing_child(&store.namespace, "workers", true)?; let binding = lifetime.binding(); let name = binding @@ -309,6 +325,7 @@ fn prepare( store, journal, intent, + spare, cancellation: None, phase, workers, @@ -330,6 +347,9 @@ impl Prepared { return Err("Snapshot retention no longer permits reclamation".into()); } self.intent.verify(&self.journal, "intent.json")?; + if let Some(spare) = &self.spare { + spare.verify(&self.journal, super::intent_slots::SPARE)?; + } if let Some(pin) = &self.cancellation { pin.verify(&self.journal, super::preparation::FILE)?; } @@ -376,6 +396,7 @@ impl Prepared { .map(|file| file.name.as_str()) .chain(std::iter::once("intent.json")) .chain(self.cancellation.iter().map(|_| super::preparation::FILE)) + .chain(record.updates.iter().map(|_| super::intent_slots::SPARE)) .collect::>(); if storage::names(&self.journal, super::ENTRY_LIMIT * 3)? .iter() diff --git a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs index 05f5410fe..8fd1c6c6d 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs @@ -114,6 +114,7 @@ fn prepare( .ok_or("Missing prepared intent digest")?; let files = payloads(&store, &journal, &record, frozen.as_ref())?; let intent = MetadataPin::open(&journal, "intent.json")?; + let spare = MetadataPin::spare(&journal, &record)?; let cancellation = Some(MetadataPin::open( &journal, super::super::preparation::FILE, @@ -133,6 +134,7 @@ fn prepare( store, journal, intent, + spare, cancellation, phase, workers, diff --git a/crates/cyber-snapshot/src/copy/reclamation/paired.rs b/crates/cyber-snapshot/src/copy/reclamation/paired.rs index 4c61fa3ad..1962e60e8 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/paired.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/paired.rs @@ -9,6 +9,7 @@ struct Owner { origin: SnapshotOrigin, journal: Dir, intent: MetadataPin, + spare: Option, digest: String, phase: MetadataPin, lifetime: super::super::lifetime::Lifetime, @@ -100,11 +101,14 @@ fn owner( )?; phase.verify(workers, name)?; let intent = MetadataPin::open(&journal, "intent.json")?; + let (_, record) = store.journal(&origin.binding.journal_id)?; + let spare = MetadataPin::spare(&journal, &record)?; Ok(Owner { checkouts, origin: origin.clone(), journal, intent, + spare, digest, phase, lifetime, @@ -199,11 +203,19 @@ fn prepare( Ok(prepared) } -fn inventory(journal: &Dir, count: usize) -> Result<(), String> { - let known = (0..count) +fn inventory( + journal: &Dir, + count: usize, + binding: Option<&super::super::intent_slots::Binding>, +) -> Result<(), String> { + let mut known = (0..count) .map(|index| format!("stage_{index}")) .chain(std::iter::once("intent.json".into())) .collect::>(); + if let Some(binding) = binding { + binding.verify(journal)?; + known.insert(super::super::intent_slots::SPARE.to_owned()); + } if let Some(name) = storage::names(journal, super::super::ENTRY_LIMIT * 3)? .into_iter() .find(|name| !known.contains(name)) @@ -248,7 +260,11 @@ fn original_payloads( if outcome.files.len() != parent.changes.len() { return Err("Incomplete recovery slot inventory".into()); } - inventory(&owner.journal, parent.changes.len())?; + inventory( + &owner.journal, + parent.changes.len(), + parent.updates.as_ref(), + )?; let mut files = Vec::new(); let planned = planned(frozen, owner); for (index, slots) in outcome.files.iter().enumerate() { @@ -276,7 +292,11 @@ fn resolution_payloads( record: &Journal, frozen: Option<&Plan>, ) -> Result, String> { - inventory(&owner.journal, record.changes.len())?; + inventory( + &owner.journal, + record.changes.len(), + record.updates.as_ref(), + )?; let ownership = record .ownership .as_ref() @@ -325,6 +345,9 @@ impl PreparedPair { owner.lifetime.verify()?; self.store.verify_worker_namespace(&owner.origin)?; owner.intent.verify(&owner.journal, "intent.json")?; + if let Some(spare) = &owner.spare { + spare.verify(&owner.journal, super::super::intent_slots::SPARE)?; + } owner.phase.verify( &self.workers, owner @@ -343,6 +366,7 @@ impl PreparedPair { if self.store.worker_digest(Some(&owner.origin))?.as_ref() != Some(&owner.digest) { return Err("Paired cleanup intent changed".into()); } + let (_, record) = self.store.journal(&owner.origin.binding.journal_id)?; let known = self .plan .files @@ -352,6 +376,12 @@ impl PreparedPair { }) .map(|file| file.name.as_str()) .chain(std::iter::once("intent.json")) + .chain( + record + .updates + .iter() + .map(|_| super::super::intent_slots::SPARE), + ) .collect::>(); if storage::names(&owner.journal, super::super::ENTRY_LIMIT * 3)? .iter() @@ -428,7 +458,7 @@ mod tests { let dir = Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(); let name = "foreign\nowner"; storage::create(&dir, name, b"unknown owner bytes").unwrap(); - let error = inventory(&dir, 0).unwrap_err(); + let error = inventory(&dir, 0, None).unwrap_err(); assert_eq!( error, format!("Unknown recovery payload or construction entry: {name:?}") diff --git a/crates/cyber-snapshot/src/copy/resolution.rs b/crates/cyber-snapshot/src/copy/resolution.rs index 6f159e9fd..b2854c1f5 100644 --- a/crates/cyber-snapshot/src/copy/resolution.rs +++ b/crates/cyber-snapshot/src/copy/resolution.rs @@ -157,7 +157,8 @@ fn resolution_id(original: &SnapshotOrigin, request: &str) -> Result { let key = serde_json::to_vec(&(original.session_created_id.as_str(), request)).map_err(fail)?; Ok(format!("restore_resolution_{}", storage::digest(&key))) } -fn immutable_digest(record: &Journal) -> Result { +pub(super) fn immutable_digest(record: &Journal) -> Result { + let format = record.format_version()?; let legacy = storage::digest( &serde_json::to_vec(&( record.version, @@ -173,11 +174,11 @@ fn immutable_digest(record: &Journal) -> Result { .map_err(fail)?, ); // Existing writer receipts keep exactly their original digest framing. - match &record.backend { - None => Ok(legacy), - Some(backend) => Ok(storage::digest( + let digest = match &record.backend { + None => legacy, + Some(backend) => storage::digest( &serde_json::to_vec(&( - if record.version == 6 { + if format == 6 { "snapshot-journal-v6" } else { "snapshot-journal-v5" @@ -186,7 +187,13 @@ fn immutable_digest(record: &Journal) -> Result { legacy, )) .map_err(fail)?, + ), + }; + match &record.updates { + Some(binding) => Ok(storage::digest( + &serde_json::to_vec(&("snapshot-journal-v7", binding, digest)).map_err(fail)?, )), + None => Ok(digest), } } diff --git a/crates/cyber-snapshot/src/copy/restore.rs b/crates/cyber-snapshot/src/copy/restore.rs index 21023fb26..72dce5b85 100644 --- a/crates/cyber-snapshot/src/copy/restore.rs +++ b/crates/cyber-snapshot/src/copy/restore.rs @@ -33,8 +33,11 @@ pub(super) struct Journal { // Version 4 binds Git context. Version 5 adds explicit backend and link kinds; // older records retain their original file-only encoding and digest framing. // Version 6 binds mixed copy/Git image provenance without relabeling IDs. + // Version 7 binds two preallocated update inodes and the original image format. pub version: u32, #[serde(default, skip_serializing_if = "Option::is_none")] + pub updates: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub backend: Option, pub location: std::path::PathBuf, pub root: String, @@ -58,6 +61,18 @@ pub(super) struct Journal { pub resolution: Option, } +impl Journal { + pub(super) fn format_version(&self) -> Result { + match (self.version, &self.updates) { + (7, Some(binding)) if self.origin.is_some() && self.ownership.is_some() => { + binding.format() + } + (1..=6, None) => Ok(self.version), + _ => Err("Invalid journal intent-slot version".into()), + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct RestoreContext { @@ -98,16 +113,20 @@ impl Store { let record: Journal = serde_json::from_slice(&storage::read(&dir, "intent.json", RECORD_LIMIT)?) .map_err(fail)?; + let format = record.format_version()?; + if let Some(binding) = &record.updates { + binding.verify(&dir)?; + } if !matches!( - (record.version, record.origin.is_some()), + (format, record.origin.is_some()), (1, false) | (2, true) | (3, _) | (4, _) | (5, _) | (6, _) - ) || (record.version == 4 + ) || (format == 4 && !record .context .as_ref() .is_some_and(|context| context.git.is_some())) - || (record.version < 5 && record.backend.is_some()) - || (record.version >= 5 + || (format < 5 && record.backend.is_some()) + || (format >= 5 && match &record.backend { Some(JournalBackend::Git) => !record .context @@ -119,10 +138,10 @@ impl Store { .is_some_and(|context| context.git.is_some()), None => true, }) - || (record.version == 6 && record.backend != Some(JournalBackend::Git)) + || (format == 6 && record.backend != Some(JournalBackend::Git)) || record.location != self.root || record.root != self.root_id - || (record.version < 5 + || (format < 5 && (record.changes.iter().any(|change| { [&change.before, &change.after] .into_iter() @@ -154,8 +173,8 @@ impl Store { .bytes() .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') }) - || (record.version < 3 && record.ownership.is_some()) - || (record.version >= 3 + || (format < 3 && record.ownership.is_some()) + || (format >= 3 && record.ownership.is_none() && (record.completed || !record.applied.is_empty())) { @@ -168,7 +187,7 @@ impl Store { resolution.validate(id, &record)?; } if let Some(context) = &record.context { - context.validate(&self.root, record.version)?; + context.validate(&self.root, format)?; } if let Some(origin) = &record.origin { let binding = &origin.binding; @@ -425,7 +444,15 @@ impl Store { .map_err(RestoreError::Failed)?; journal.origin = origin; - Self::record(&dir, journal).map_err(RestoreError::Failed)?; + let construction = if worker.is_some() { + Some( + super::intent_slots::Construction::create(&dir, journal) + .map_err(RestoreError::Failed)?, + ) + } else { + Self::record(&dir, journal).map_err(RestoreError::Failed)?; + None + }; let preimages = self .resolution_preimages(journal.resolution.as_ref()) .map_err(RestoreError::Failed)?; @@ -452,7 +479,19 @@ impl Store { storage::sync(&dir).map_err(RestoreError::Failed)?; journal.ownership = Some(Ownership::prepare(self, &dir, &journal.changes).map_err(RestoreError::Failed)?); - Self::record(&dir, journal).map_err(RestoreError::Failed)?; + if let Some(construction) = &construction { + journal.updates = Some( + construction + .binding(&dir, journal.version) + .map_err(RestoreError::Failed)?, + ); + journal.version = 7; + construction + .publish(&dir, journal) + .map_err(RestoreError::Failed)?; + } else { + Self::record(&dir, journal).map_err(RestoreError::Failed)?; + } if let Some(preimages) = &preimages { preimages .verify_review(self) @@ -585,6 +624,7 @@ impl Store { }; let mut journal = Journal { version, + updates: None, backend: (version >= 5).then(|| { if context .as_ref() @@ -610,6 +650,10 @@ impl Store { context, resolution, }; + if worker.is_some() { + journal.updates = Some(super::intent_slots::Binding::budget(version)); + journal.version = 7; + } let bytes = serde_json::to_vec(&journal).map_err(fail)?.len() as u64; if bytes > RECORD_LIMIT { return Err("Restore journal exceeds limit".into()); @@ -645,6 +689,8 @@ impl Store { |sum, length| sum.checked_add(length).ok_or("Restore size overflow"), )?; self.reserve(reserve)?; + journal.version = version; + journal.updates = None; journal.applied.clear(); journal.origin = None; journal.acknowledged = false; @@ -766,6 +812,10 @@ impl Store { storage::sync_file(file) } pub(super) fn record(dir: &cap_std::fs::Dir, journal: &Journal) -> Result<()> { + journal.format_version()?; + if journal.updates.is_some() { + return super::intent_slots::write(dir, journal); + } let bytes = serde_json::to_vec(journal).map_err(fail)?; if bytes.len() as u64 > RECORD_LIMIT { return Err("Restore journal exceeds limit".into()); @@ -1003,6 +1053,7 @@ mod tests { &Journal { backend: None, version: 2, + updates: None, location: store.root.clone(), root: store.root_id.clone(), created: now(), diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 72e2ffacf..e1cb168de 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -1,6 +1,6 @@ # Active implementation goal and delivery rules -## Current audit: 2026-11-09 +## Current audit: 2026-11-10 The user has set the active objective to complete implementation, verification and accepted delivery of **#93**. #92 remains a separate unfinished workstream; @@ -77,6 +77,26 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. on earlier recovery success. This is an open construction/acceptance failure, not a timeout or an accepted native result. +6. **Owned intent updates now avoid untracked source-phase temporary files.** + Version-7 prepared journals bind two preallocated native inodes before source + admission. Updates synchronize the spare, exchange the known names and sync + the directory; reopen and cleanup verify both identities. Cleanup also pins + both metadata generations across its durable plan and every payload unlink. + A real subprocess kill during a torn spare write preserves the active record + and permits exact metadata continuation after reopening. Legacy unknown files + remain fenced; this is not complete construction or metadata retirement. + The broad parallel CLI run failed three old-version assertions (corrected), + two preparation observation deadlines and one typed storage-busy TUI response. + The three preparation/TUI cases pass serially; this does not establish the + cause of the parallel failures. The complete serial CLI rerun is pending. + +Native run [42733865471](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42733865471) +passes Linux x64/ARM and macOS ARM/Intel at `e2b3ba18`, including the earlier +managed Git cleanup and structured CLI cases. It predates the owned intent-slot +changes and is not evidence for them. The workflow now also selects the three +pre-effect preparation/TUI client cases serially and retains their log; that +workflow addition still requires source-bound platform acceptance. + Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; formatting and whitespace diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index cd97d379d..3736240fc 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -147,6 +147,27 @@ native proof, admit a recovery operation or implement a resolution receipt. ## Safe reclamation +### Owned intent update slots + +New runtime-owned prepared journals SHALL use version 7, retaining the original +copy/Git image format as an explicit base version. Before source admission the +worker SHALL create and retain two unique regular intent inodes, synchronize +them, and publish their native creation identities in the immutable prepared +intent. This binding SHALL enter the consuming intent digest. Progress, +completion and acknowledgement updates SHALL overwrite only the retained spare +inode, synchronize it, atomically exchange the two known names, and synchronize +the journal directory. Source-phase updates SHALL NOT allocate anonymous +temporary intent files. + +The active intent SHALL always contain a complete published record; interrupted +spare contents are mutable metadata, never source-effect or acknowledgement +authority. Reopen and every update SHALL verify both exact unique inode routes. +Reclamation SHALL retain both metadata slots and recognize the spare only with +this immutable ownership proof. Missing, replaced or aliased slots SHALL refuse; +existing journals and unknown temporary/construction files gain no ownership. +Pre-admission construction without durable slot identities and other worker or +cancellation-record construction cuts remain independently fenced. + Only records with both the durable owner resolution receipt and matching journal acknowledgement are eligible. Reference/reader accounting protects staged baselines, inactive branches, leases, binary saved versions and any unresolved intent. A missing diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index 2d4c807db..bdf299be7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -504,3 +504,10 @@ - **AND** unknown object entries and unsupported temporary-file aliases SHALL refuse before any manifest unlink; an already empty namespace SHALL remain a no-op - **AND** disabled snapshots or zero retention SHALL preserve those bytes, and orphan/temporary aging SHALL use the configured retention duration - **AND** a changed policy SHALL refuse before the next unlink, preserving remaining manifests and objects even if earlier eligible manifests were already removed + +#### Scenario: Owned intent update is interrupted after source admission +- **GIVEN** the immutable prepared intent binds two preallocated native intent inodes before source admission +- **WHEN** a worker stops while writing progress, completion or acknowledgement into the spare slot +- **THEN** the active intent SHALL remain complete and its native slot binding SHALL permit inspection or exact metadata reconciliation without trusting partial spare bytes +- **AND** source-phase updates SHALL create no anonymous temporary intent file; cleanup SHALL retain both known metadata slots +- **AND** equal-byte replacement inodes, aliases, missing slots and legacy unknown entries SHALL refuse without deletion or source replay diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 98fc17bf2..9d9566e0c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -933,3 +933,38 @@ Independent JSONL/structured checks run unless cancelled even after an earlier recovery failure, without relaxing the overall job gate. The native workflow also selects the complete App reclamation suite. These edits provide diagnostics and coverage, not a fix or passing acceptance for the macOS construction failure. + +Owned intent-update continuation (2026-11-10): runtime-owned prepared journals +now publish version 7 with an immutable binding to two preallocated native intent +inodes and the original copy/Git image format. Progress, completion and native +acknowledgement overwrite/sync the spare, exchange the two names and sync the +directory; they allocate no anonymous source-phase temporary intent. Reopen and +updates reject missing, replaced, aliased or invalid bindings. Ordinary, paired +and prepared-cancellation payload cleanup retain both slots, recognize the spare +only through its binding and recheck both metadata generations before unlinks. +Legacy digest framing and unknown construction fences remain unchanged. + +Local evidence: five intent-slot cases pass, including an actual subprocess kill +during a torn spare write and physical Store reopen; its ignored child is invoked +by the parent proof. The complete snapshot candidate run passes 197 cases with +eight intentional subprocess/diagnostic skips, before the final malformed-version +writer guard; the slot cases pass again with that guard. Eighteen App reclamation +and two App Git runtime cases pass with both metadata pins. Three real managed Git +CLI cases pass before the extra spare-generation pin; the full final serial CLI +run is pending. Four non-Git CLI cases pass after correcting the old version-3 +assertions. The broad four-thread CLI run records six failures: those three +version assertions, two pre-origin observation deadlines and one typed storage-busy +TUI acknowledgement. The three preparation/TUI cases pass serially in 124.39s; +the parallel failure causes remain unestablished, and no safety fence or deadline +was relaxed. Six structured CLI cases pass, including a separate timeout with a +verified in-flight correction; short deadlines may legitimately expire before +main inference. All-target snapshot/App/CLI Clippy passes after extracting the +copy-intent header assertions. Focused code/security review finds no additional +issue in this increment; full-branch review and delivery acceptance remain open. + +Native run 42733865471 passes all four Linux/macOS jobs at e2b3ba18. It predates +these changes and accepts only that checkpoint. The workflow now selects the +preparation/TUI client cases serially in addition to managed Git recovery; fresh +source-bound platform evidence remains required. Partial construction, complete +reference/CAS/metadata retirement, default Git admission and M2–M8/parent task +acceptance remain open. No public API or SDK schema changes in this increment. From 124c11b969b089ecf81597f814296deced268857 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 00:34:28 -0400 Subject: [PATCH 61/87] Retain collection policy provenance across interrupted pruning --- crates/cyber-snapshot/src/copy/collection.rs | 104 +++-- .../src/copy/collection/provenance.rs | 417 ++++++++++++++++++ crates/cyber-snapshot/src/copy/mod.rs | 28 +- crates/cyber-snapshot/tests/file_copy.rs | 133 ++++++ docs/goal-delivery.md | 20 +- .../managed-recovery.md | 18 +- .../specs/snapshots-checkpoints/spec.md | 9 + .../complete-daily-coding-surfaces/tasks.md | 38 ++ 8 files changed, 722 insertions(+), 45 deletions(-) create mode 100644 crates/cyber-snapshot/src/copy/collection/provenance.rs diff --git a/crates/cyber-snapshot/src/copy/collection.rs b/crates/cyber-snapshot/src/copy/collection.rs index 945cb40b6..38b5f4fc6 100644 --- a/crates/cyber-snapshot/src/copy/collection.rs +++ b/crates/cyber-snapshot/src/copy/collection.rs @@ -1,12 +1,9 @@ //! Policy-bound fallback collection; unresolved worker/journal evidence stays fenced. use super::{ENTRY_LIMIT, ImageKind, Manifest, RECORD_LIMIT, now, storage}; use cap_std::fs::Dir; -use std::{ - collections::BTreeSet, - path::{Path, PathBuf}, - sync::Arc, -}; +use std::{collections::BTreeSet, path::Path, sync::Arc}; use storage::{Result, fail}; +mod provenance; /// Prune only unreferenced, old manifests. Journals keep all their objects pinned; /// an unavailable durable-reference view never authorizes deletion. @@ -27,6 +24,7 @@ pub(crate) fn gc( return Ok(()); } let root = storage::existing_child(&snapshot, "global", true)?; + verify_references(&root, protected)?; for location in storage::names(&root, 100_000)? { if !storage::valid_digest(&location) { return Err("Invalid snapshot namespace".into()); @@ -40,6 +38,42 @@ pub(crate) fn gc( } Ok(()) } +/// A durable root whose manifest has disappeared has an unknown object set. +/// Resolve all protected roots before any namespace can collect orphan objects. +fn verify_references(root: &Dir, protected: &BTreeSet) -> Result<()> { + if protected.len() > 100_000 { + return Err("Snapshot collection reference limit exceeded".into()); + } + let mut pending = protected + .iter() + .filter(|id| id.starts_with("copy:")) + .map(|id| { + id.strip_prefix("copy:") + .filter(|id| storage::valid_digest(id)) + .ok_or_else(|| "Invalid snapshot collection reference".to_string()) + }) + .collect::>>()?; + if pending.is_empty() { + return Ok(()); + } + for namespace in storage::names(root, 100_000)? { + if !storage::valid_digest(&namespace) { + return Err("Invalid snapshot namespace".into()); + } + let dir = storage::existing_child(root, &namespace, true)?; + let manifests = storage::existing_child(&dir, "manifests", true)?; + for name in storage::names(&manifests, 100_000)? { + if pending.contains(name.as_str()) { + read_manifest(&manifests, &name, &namespace)?; + pending.remove(name.as_str()); + } + } + if pending.is_empty() { + return Ok(()); + } + } + Err("Snapshot collection reference manifest is unavailable".into()) +} pub(super) fn gc_location( dir: &Dir, namespace: &str, @@ -67,10 +101,12 @@ pub(super) fn gc_location( if names.is_empty() && storage::names(&objects, 100_000)?.is_empty() { return Ok(()); } - let Some(retention) = Retention::inspect(&manifests, &names, namespace, config)? else { + let provenance = provenance::Provenance::inspect(dir, &manifests, &names, namespace)?; + let Some(mut retention) = Retention::inspect(provenance, config)? else { return Ok(()); }; let object_names = object_inventory(&objects)?; + retention.provenance.publish()?; let mut keep = BTreeSet::new(); for name in names { if temporary(&name) { @@ -119,56 +155,42 @@ pub(super) fn gc_location( /// A complete validated manifest inventory identifies the Location even when its /// source directory is gone. Never infer a default policy from orphan names. struct Retention { - location: PathBuf, + provenance: provenance::Provenance, settings: serde_json::Value, days: u64, } impl Retention { fn inspect( - manifests: &Dir, - names: &[String], - namespace: &str, + provenance: provenance::Provenance, config: &crate::ConfigFn, ) -> Result> { - let mut policy: Option = None; - for name in names { - if temporary(name) { - storage::open(manifests, Path::new(name))?; - continue; - } - let manifest = read_manifest(manifests, name, namespace)?; - if let Some(policy) = &policy { - if policy.location != manifest.tree.location { - return Err("Snapshot collection has inconsistent Location evidence".into()); - } - continue; - } - let settings = config(&manifest.tree.location); - let days = settings - .pointer("/snapshots/retention_days") - .and_then(serde_json::Value::as_u64) - .unwrap_or(7); - if settings.get("snapshots") == Some(&serde_json::Value::Bool(false)) || days == 0 { - return Ok(None); - } - policy = Some(Self { - location: manifest.tree.location, - settings, - days, - }); + let settings = config(provenance.location()); + // The App configuration adapter returns null when resolution fails. + // Missing policy authority cannot select the default retention period. + if !settings.is_object() { + return Err("Snapshot retention configuration is unavailable".into()); + } + let days = settings + .pointer("/snapshots/retention_days") + .and_then(serde_json::Value::as_u64) + .unwrap_or(7); + if settings.get("snapshots") == Some(&serde_json::Value::Bool(false)) || days == 0 { + return Ok(None); } - policy - .map(Some) - .ok_or_else(|| "Snapshot collection lacks Location policy evidence".into()) + Ok(Some(Self { + provenance, + settings, + days, + })) } fn seconds(&self) -> u64 { self.days.saturating_mul(86400) } fn verify(&self, config: &crate::ConfigFn) -> Result<()> { - if config(&self.location) != self.settings { + if config(self.provenance.location()) != self.settings { return Err("Snapshot retention policy changed during collection".into()); } - Ok(()) + self.provenance.verify() } } fn object_inventory(objects: &Dir) -> Result> { diff --git a/crates/cyber-snapshot/src/copy/collection/provenance.rs b/crates/cyber-snapshot/src/copy/collection/provenance.rs new file mode 100644 index 000000000..b53dca1f8 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/collection/provenance.rs @@ -0,0 +1,417 @@ +//! Durable policy provenance, published before the first manifest unlink. +use super::{Manifest, read_manifest, storage}; +use cap_std::fs::Dir; +use serde::{Deserialize, Serialize}; +use std::{fs::File, io::Read, path::Path}; +use storage::{Result, fail}; + +const FILE: &str = "collection.json"; +const LIMIT: u64 = 64 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Record { + version: u32, + location: std::path::PathBuf, + root: String, + namespace: String, +} +impl Record { + fn from_manifest(dir: &Dir, manifest: Manifest) -> Result { + Ok(Self { + version: 1, + location: manifest.tree.location, + root: manifest.tree.root, + namespace: storage::dir_identity(dir)?, + }) + } + fn validate(&self, dir: &Dir, namespace: &str) -> Result<()> { + let key = storage::digest(format!("{}\0{}", self.location.display(), self.root).as_bytes()); + if self.version != 1 + || !self.location.is_absolute() + || self.location.parent().is_none() + || !super::super::ownership::valid_identity(&self.root) + || self.namespace != storage::dir_identity(dir)? + || key != namespace + { + return Err("Corrupt or foreign snapshot collection provenance".into()); + } + Ok(()) + } +} + +struct Pin { + file: File, + identity: String, + generation: String, +} +impl Pin { + fn new(file: File) -> Result { + storage::validate_lock(&file)?; + Ok(Self { + identity: storage::identity(&file)?, + generation: super::super::ownership::generation(&file)?, + file, + }) + } + fn verify(&self, dir: &Dir) -> Result<()> { + let route = storage::open(dir, Path::new(FILE))?; + storage::validate_lock(&route)?; + if storage::identity(&route)? != self.identity + || super::super::ownership::generation(&route)? != self.generation + || super::super::ownership::generation(&self.file)? != self.generation + { + return Err("Snapshot collection provenance identity or generation changed".into()); + } + Ok(()) + } +} + +pub(super) struct Provenance { + record: Record, + directory: Dir, + pin: Option, +} +impl Provenance { + pub fn inspect(dir: &Dir, manifests: &Dir, names: &[String], namespace: &str) -> Result { + let retained = match dir.symlink_metadata(FILE) { + Ok(_) => Some(Self::load(dir, namespace)?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(fail(error)), + }; + let mut record = retained.as_ref().map(|(record, _)| record.clone()); + // Preflight every surviving manifest, including after a prior collection. + for name in names { + if super::temporary(name) { + storage::open(manifests, Path::new(name))?; + continue; + } + let observed = Record::from_manifest(dir, read_manifest(manifests, name, namespace)?)?; + observed.validate(dir, namespace)?; + if record.as_ref().is_some_and(|record| *record != observed) { + return Err("Snapshot collection has inconsistent Location evidence".into()); + } + record = Some(observed); + } + Ok(Self { + record: record.ok_or("Snapshot collection lacks Location policy evidence")?, + directory: dir.try_clone().map_err(fail)?, + pin: retained.map(|(_, pin)| pin), + }) + } + fn load(dir: &Dir, namespace: &str) -> Result<(Record, Pin)> { + let pin = Pin::new(storage::open(dir, Path::new(FILE))?)?; + if pin.file.metadata().map_err(fail)?.len() > LIMIT { + return Err("Snapshot collection provenance exceeds limit".into()); + } + let mut bytes = Vec::new(); + (&pin.file) + .take(LIMIT + 1) + .read_to_end(&mut bytes) + .map_err(fail)?; + if bytes.len() as u64 > LIMIT { + return Err("Snapshot collection provenance grew beyond limit".into()); + } + pin.verify(dir)?; + let record: Record = serde_json::from_slice(&bytes).map_err(fail)?; + record.validate(dir, namespace)?; + Ok((record, pin)) + } + pub fn location(&self) -> &Path { + &self.record.location + } + pub fn publish(&mut self) -> Result<()> { + if self.pin.is_some() { + return self.verify(); + } + let bytes = serde_json::to_vec(&self.record).map_err(fail)?; + if bytes.len() as u64 > LIMIT { + return Err("Snapshot collection provenance exceeds limit".into()); + } + let temporary = format!("tmp_{}", ulid::Ulid::new()); + let file = storage::create(&self.directory, &temporary, &bytes)?; + rustix::fs::renameat_with( + &self.directory, + &temporary, + &self.directory, + FILE, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(fail)?; + storage::sync(&self.directory)?; + self.pin = Some(Pin::new(file)?); + self.verify() + } + pub fn verify(&self) -> Result<()> { + self.pin + .as_ref() + .ok_or("Snapshot collection provenance was not published")? + .verify(&self.directory) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::copy::{Options, Store, collection::gc_location}; + use std::collections::BTreeSet; + + fn reopen(base: &Path) -> Store { + Store::recover_existing( + &base.join("data"), + base.join("work").to_str().unwrap(), + options(), + ) + .unwrap() + } + fn options() -> Options { + Options { + max_file: 65536, + max_total: 4 * 1024 * 1024, + ignore: vec![], + private_roots: vec![], + } + } + fn key(store: &Store) -> String { + storage::digest(format!("{}\0{}", store.root.display(), store.root_id).as_bytes()) + } + fn fixture() -> (tempfile::TempDir, Store) { + let temp = tempfile::tempdir().unwrap(); + let work = temp.path().join("work"); + std::fs::create_dir(&work).unwrap(); + std::fs::write(work.join("owner"), b"owner bytes").unwrap(); + let store = + Store::open(&temp.path().join("data"), work.to_str().unwrap(), options()).unwrap(); + store.track().unwrap(); + for name in storage::names(&store.manifests, 10).unwrap() { + let mut manifest: Manifest = serde_json::from_slice( + &storage::read(&store.manifests, &name, super::super::RECORD_LIMIT).unwrap(), + ) + .unwrap(); + manifest.accessed = 0; + storage::atomic( + &store.manifests, + &name, + &serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + } + for name in storage::names(&store.objects, 10).unwrap() { + storage::open(&store.objects, Path::new(&name)) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH)) + .unwrap(); + } + (temp, store) + } + fn publish(store: &Store) -> Provenance { + let names = storage::names(&store.manifests, 10).unwrap(); + let mut proof = + Provenance::inspect(&store.namespace, &store.manifests, &names, &key(store)).unwrap(); + proof.publish().unwrap(); + proof + } + fn collect(store: &Store, config: &crate::ConfigFn) -> Result<()> { + gc_location(&store.namespace, &key(store), config, &BTreeSet::new()) + } + + #[test] + fn a_foreign_or_malformed_record_cannot_be_overwritten_from_manifest_evidence() { + let (_temp, store) = fixture(); + let (_foreign_temp, foreign) = fixture(); + let foreign_proof = publish(&foreign); + storage::create( + &store.namespace, + FILE, + &serde_json::to_vec(&foreign_proof.record).unwrap(), + ) + .unwrap(); + for invalid in [false, true] { + if invalid { + store.namespace.write(FILE, b"{").unwrap(); + } + let bytes = storage::read(&store.namespace, FILE, LIMIT).unwrap(); + assert!(collect(&store, &|_| serde_json::json!({})).is_err()); + assert_eq!(storage::read(&store.namespace, FILE, LIMIT).unwrap(), bytes); + assert_eq!(storage::names(&store.manifests, 10).unwrap().len(), 1); + assert_eq!(storage::names(&store.objects, 10).unwrap().len(), 1); + } + } + + #[test] + fn metadata_replacement_after_preflight_refuses_before_unlinks() { + for replace in [false, true] { + changed_metadata_refuses(replace); + } + } + fn changed_metadata_refuses(replace: bool) { + let (_temp, store) = fixture(); + let proof = publish(&store); + let bytes = serde_json::to_vec(&proof.record).unwrap(); + let called = std::sync::atomic::AtomicUsize::new(0); + let directory = store.namespace.try_clone().unwrap(); + let result = collect(&store, &move |_| { + if called.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 1 { + if replace { + directory + .rename(FILE, &directory, "retained-record") + .unwrap(); + storage::create(&directory, FILE, &bytes).unwrap(); + } else { + let mut changed = bytes.clone(); + changed.push(b'\n'); + directory.write(FILE, changed).unwrap(); + } + } + serde_json::json!({}) + }); + assert!( + result + .unwrap_err() + .contains("identity or generation changed") + ); + assert_eq!(storage::names(&store.manifests, 10).unwrap().len(), 1); + assert_eq!(storage::names(&store.objects, 10).unwrap().len(), 1); + } + + #[test] + fn provenance_cannot_be_transplanted_into_another_native_namespace() { + let (temp, store) = fixture(); + let proof = publish(&store); + let other = Dir::open_ambient_dir(temp.path(), cap_std::ambient_authority()).unwrap(); + let other = storage::private_child(&other, "foreign-namespace").unwrap(); + storage::create(&other, FILE, &serde_json::to_vec(&proof.record).unwrap()).unwrap(); + assert!(Provenance::load(&other, &key(&store)).is_err()); + assert_eq!(storage::names(&store.manifests, 10).unwrap().len(), 1); + } + + #[test] + fn provenance_aliases_never_authorize_collection() { + for symbolic in [false, true] { + let (_temp, store) = fixture(); + let proof = publish(&store); + store + .namespace + .rename(FILE, &store.namespace, "retained-record") + .unwrap(); + if symbolic { + rustix::fs::symlinkat("retained-record", &store.namespace, FILE).unwrap(); + } else { + store + .namespace + .hard_link("retained-record", &store.namespace, FILE) + .unwrap(); + } + assert!(collect(&store, &|_| serde_json::json!({})).is_err()); + assert_eq!(storage::names(&store.manifests, 10).unwrap().len(), 1); + assert_eq!(storage::names(&store.objects, 10).unwrap().len(), 1); + let retained: Record = + serde_json::from_slice(&store.namespace.read("retained-record").unwrap()).unwrap(); + assert_eq!(retained, proof.record); + } + } + + #[test] + fn namespace_metadata_and_unpublished_construction_count_toward_quota() { + let (temp, store) = fixture(); + let before = store.namespace_metadata_size().unwrap(); + let proof = publish(&store); + let record_size = proof.pin.as_ref().unwrap().file.metadata().unwrap().len(); + storage::create(&store.namespace, "tmp_unpublished", &[0; 2048]).unwrap(); + assert_eq!( + store.namespace_metadata_size().unwrap(), + before + record_size + 2048 + ); + drop(proof); + drop(store); + let reopened = reopen(temp.path()); + assert!(reopened.usage().unwrap() >= record_size + 2048); + } + + #[test] + #[ignore = "subprocess fixture invoked by collection-restart proof"] + fn collection_child() { + let base = std::path::PathBuf::from(std::env::var_os("CYBER_COLLECTION_CHILD").unwrap()); + let store = reopen(&base); + let manifests = store.manifests.try_clone().unwrap(); + collect(&store, &move |_| { + if storage::names(&manifests, 10).unwrap().is_empty() { + std::fs::write(base.join("ready"), b"last manifest gone, objects retained") + .unwrap(); + loop { + std::thread::sleep(std::time::Duration::from_secs(1)); + } + } + serde_json::json!({}) + }) + .unwrap(); + panic!("child passed the cut"); + } + + #[test] + fn process_death_after_last_manifest_unlink_resumes_from_durable_policy_provenance() { + use std::os::unix::process::ExitStatusExt; + let (temp, original) = fixture(); + drop(original); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "copy::collection::provenance::tests::collection_child", + "--ignored", + "--nocapture", + ]) + .env("CYBER_COLLECTION_CHILD", temp.path()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + let ready = loop { + if temp.path().join("ready").exists() { + break true; + } + if child.try_wait().unwrap().is_some() || std::time::Instant::now() >= deadline { + break false; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + }; + let _ = child.kill(); + let status = child.wait().unwrap(); + assert!(ready, "child missed the collection cut: {status}"); + assert_eq!(status.signal(), Some(9)); + let reopened = reopen(temp.path()); + assert!(storage::names(&reopened.manifests, 10).unwrap().is_empty()); + assert_eq!(storage::names(&reopened.objects, 10).unwrap().len(), 1); + reopened + .namespace + .rename(FILE, &reopened.namespace, "retained-record") + .unwrap(); + assert!(collect(&reopened, &|_| serde_json::json!({})).is_err()); + assert_eq!(storage::names(&reopened.objects, 10).unwrap().len(), 1); + reopened + .namespace + .rename("retained-record", &reopened.namespace, FILE) + .unwrap(); + assert_preserved_by_unavailable_and_disabled_policy(&reopened); + collect(&reopened, &|_| serde_json::json!({})).unwrap(); + assert!(storage::names(&reopened.objects, 10).unwrap().is_empty()); + assert!(reopened.namespace.symlink_metadata(FILE).unwrap().is_file()); + assert_eq!( + std::fs::read(temp.path().join("work/owner")).unwrap(), + b"owner bytes" + ); + } + fn assert_preserved_by_unavailable_and_disabled_policy(store: &Store) { + assert!( + collect(store, &|_| serde_json::Value::Null) + .unwrap_err() + .contains("configuration is unavailable") + ); + assert_eq!(storage::names(&store.objects, 10).unwrap().len(), 1); + for settings in [ + serde_json::json!({"snapshots":false}), + serde_json::json!({"snapshots":{"retention_days":0}}), + ] { + collect(store, &move |_| settings.clone()).unwrap(); + assert_eq!(storage::names(&store.objects, 10).unwrap().len(), 1); + } + } +} diff --git a/crates/cyber-snapshot/src/copy/mod.rs b/crates/cyber-snapshot/src/copy/mod.rs index 1ace9a7ec..1b26fbe30 100644 --- a/crates/cyber-snapshot/src/copy/mod.rs +++ b/crates/cyber-snapshot/src/copy/mod.rs @@ -293,7 +293,7 @@ impl Store { if let Some(size) = self.usage.get() { return Ok(size); } - let mut size = 0u64; + let mut size = self.namespace_metadata_size()?; for dir in [&self.objects, &self.manifests] { for name in storage::names(dir, 100_000)? { size = size @@ -348,6 +348,32 @@ impl Store { self.usage.set(Some(size)); Ok(size) } + fn namespace_metadata_size(&self) -> Result { + let mut size = 0u64; + for name in storage::names(&self.namespace, ENTRY_LIMIT)? { + let metadata = self.namespace.symlink_metadata(&name).map_err(fail)?; + if metadata.is_dir() { + if !matches!( + name.as_str(), + "files" | "manifests" | "journals" | "workers" + ) { + return Err("Unexpected snapshot namespace directory".into()); + } + continue; + } + // Include durable collection provenance and unpublished construction + // bytes. Unknown files stay retained; names never grant cleanup. + size = size + .checked_add( + storage::open(&self.namespace, Path::new(&name))? + .metadata() + .map_err(fail)? + .len(), + ) + .ok_or("Snapshot storage overflow")?; + } + Ok(size) + } fn reserve(&self, size: u64) -> Result<()> { if self.options.max_total == 0 || self diff --git a/crates/cyber-snapshot/tests/file_copy.rs b/crates/cyber-snapshot/tests/file_copy.rs index ff9800fff..fbd85de31 100644 --- a/crates/cyber-snapshot/tests/file_copy.rs +++ b/crates/cyber-snapshot/tests/file_copy.rs @@ -845,6 +845,14 @@ async fn collection_zero_retention_preserves_orphan_and_temporary_bytes() { assert_collection_retained(&retained); } #[tokio::test] +async fn collection_unavailable_configuration_preserves_the_complete_inventory() { + let (p, retained) = aged_collection_fixture().await; + *p.config.lock().unwrap() = Value::Null; + p.snapshots.gc().await; + assert_collection_retained(&retained); + assert!(!namespace(&p).join("collection.json").exists()); +} +#[tokio::test] async fn collection_missing_location_evidence_preserves_orphans() { let (p, mut retained) = aged_collection_fixture().await; retained.retain(|(path, _)| { @@ -966,6 +974,22 @@ async fn collection_policy_revocation_after_manifest_deletion_preserves_objects( .filter(|(path, _)| path.parent().unwrap().ends_with("files")) .collect(); assert_collection_retained(&objects); + // Simulate a fresh collector after the stop between manifest and object + // removal. It must use durable Location evidence rather than orphan names. + p.snapshots = GitSnapshots::with_policy( + p.data.clone(), + Arc::new(|_| json!({})), + vec![], + Some(Arc::new(|| Ok(Default::default()))), + ); + p.snapshots.gc().await; + for (path, _) in objects { + assert!( + !path.exists(), + "old orphan survived restart: {}", + path.display() + ); + } } #[tokio::test] @@ -996,3 +1020,112 @@ async fn collection_unknown_object_refuses_before_any_manifest_is_removed() { assert_collection_retained(&retained); assert_eq!(std::fs::read(unknown).unwrap(), b"unknown owner"); } + +#[tokio::test] +async fn collection_dangling_reference_preserves_orphans_despite_valid_provenance() { + let (mut p, retained) = aged_collection_fixture().await; + let manifest = retained + .iter() + .find(|(path, _)| { + path.parent().unwrap().ends_with("manifests") + && !path + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("tmp_") + }) + .unwrap() + .0 + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + p.snapshots.gc().await; + let objects = restore_old_orphans(&retained); + let provenance = std::fs::read(namespace(&p).join("collection.json")).unwrap(); + p.snapshots = GitSnapshots::with_policy( + p.data.clone(), + Arc::new(|_| json!({})), + vec![], + Some(Arc::new(move || { + Ok(std::collections::BTreeSet::from([format!( + "copy:{manifest}" + )])) + })), + ); + p.snapshots.gc().await; + assert_collection_retained(&objects); + assert_eq!( + std::fs::read(namespace(&p).join("collection.json")).unwrap(), + provenance + ); +} + +#[tokio::test] +async fn collection_resolves_other_location_roots_before_reclaiming_old_orphans() { + let (mut p, retained) = aged_collection_fixture().await; + let original_namespace = namespace(&p); + p.snapshots.gc().await; + let objects = restore_old_orphans(&retained); + let other = p.root.parent().unwrap().join("other-work"); + std::fs::create_dir(&other).unwrap(); + std::fs::write(other.join("owner"), b"other Location bytes").unwrap(); + let protected = p + .snapshots + .track(other.to_str().unwrap()) + .await + .unwrap() + .unwrap(); + let root = protected.tree.clone(); + p.snapshots = GitSnapshots::with_policy( + p.data.clone(), + Arc::new(|_| json!({})), + vec![], + Some(Arc::new(move || { + Ok(std::collections::BTreeSet::from([ + root.clone(), + "a".repeat(40), + ])) + })), + ); + p.snapshots.gc().await; + for (path, _) in objects { + assert!(!path.exists(), "{}", path.display()); + } + assert!( + std::fs::read_dir(original_namespace.join("files")) + .unwrap() + .next() + .is_none() + ); + assert_eq!( + std::fs::read(other.join("owner")).unwrap(), + b"other Location bytes" + ); + assert!( + p.snapshots + .changed(other.to_str().unwrap(), &protected.tree, &protected.tree) + .await + .unwrap() + .is_empty() + ); +} + +fn restore_old_orphans(retained: &[(PathBuf, Vec)]) -> Vec<(PathBuf, Vec)> { + retained + .iter() + .filter(|(path, _)| path.parent().unwrap().ends_with("files")) + .map(|(path, bytes)| { + std::fs::write(path, bytes).unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH)) + .unwrap(); + (path.clone(), bytes.clone()) + }) + .collect() +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index e1cb168de..b3f58f980 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -88,7 +88,22 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. The broad parallel CLI run failed three old-version assertions (corrected), two preparation observation deadlines and one typed storage-busy TUI response. The three preparation/TUI cases pass serially; this does not establish the - cause of the parallel failures. The complete serial CLI rerun is pending. + cause of the parallel failures. The complete serial CLI rerun at `2ec1b1c5` + passes all 22 cases in 837.69s, including both managed Git process-death cases + and TUI reconciliation. SDK generation/typecheck and 136 SDK tests also pass + at that checkpoint, with ten configured external skips. + +7. **Collection now retains durable policy provenance across manifest deletion.** + A bounded immutable Location/root/native-namespace record is synchronized before + deletion and pinned across every unlink. A fresh collector can resume orphan + cleanup after the last manifest disappears, while disabled/zero retention, + unavailable configuration, unknown provenance and metadata changes preserve + remaining bytes. Every protected copy root must resolve to a valid manifest + somewhere in the private global inventory before collection; valid roots in + another Location do not block unrelated orphan cleanup. Namespace metadata and + unpublished construction bytes now count toward quota. Real process-kill, + restart, alias, namespace-transplant and reference regressions pass locally. + This does not retire journal/worker/checkout metadata or complete M6. Native run [42733865471](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42733865471) passes Linux x64/ARM and macOS ARM/Intel at `e2b3ba18`, including the earlier @@ -96,6 +111,9 @@ managed Git cleanup and structured CLI cases. It predates the owned intent-slot changes and is not evidence for them. The workflow now also selects the three pre-effect preparation/TUI client cases serially and retains their log; that workflow addition still requires source-bound platform acceptance. +Run [42753722365](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42753722365) +at `2ec1b1c5` has passed both Linux jobs and macOS ARM; macOS Intel is still +running at this checkpoint. It predates the collection-provenance continuation. Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 3736240fc..0f388d19b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -184,10 +184,24 @@ Fallback manifest/object collection SHALL establish the Location's current retention policy from a complete valid, namespace-bound manifest inventory before deletion. Disabled snapshots and zero retention preserve manifests, orphan objects and temporary files. Configured retention applies to orphan/temporary aging too; -policy changes refuse before the next unlink. Corrupt/foreign manifests, unknown +Unavailable configuration SHALL preserve bytes rather than select default +retention; policy changes refuse before the next unlink. Corrupt/foreign manifests, unknown object entries and temporary aliases refuse before manifest deletion. An empty namespace is a no-op. Orphans without surviving Location evidence remain retained; -durable collection/retirement evidence for those namespaces remains part of M6. +before deleting a namespace's first manifest, collection SHALL publish and sync a +bounded immutable Location/root/native-namespace record derived from the complete +valid manifest inventory. This record SHALL preserve policy provenance after the +last manifest is removed, including across process restart. Later collection SHALL +validate its namespace binding, agree with every surviving manifest, obtain fresh +policy and complete references, and recheck the exact metadata route/generation +before each unlink. Missing, malformed, transplanted or aliased records SHALL NOT +invent Location authority or be overwritten. Unpublished construction files remain +retained and count toward quota. Before any namespace collection, every durable +protected root SHALL resolve to a valid manifest in the private global inventory; +a missing referenced manifest has an unknown object set and SHALL refuse collection. +Valid roots in other Locations SHALL NOT by themselves prevent orphan reclamation. +This record does not retire journal, worker or +request metadata; complete retirement remains part of M6. ## Managed checkout payload reclamation diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index bdf299be7..e6e1a99f0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -511,3 +511,12 @@ - **THEN** the active intent SHALL remain complete and its native slot binding SHALL permit inspection or exact metadata reconciliation without trusting partial spare bytes - **AND** source-phase updates SHALL create no anonymous temporary intent file; cleanup SHALL retain both known metadata slots - **AND** equal-byte replacement inodes, aliases, missing slots and legacy unknown entries SHALL refuse without deletion or source replay + +#### Scenario: Collection restarts after removing the last manifest +- **GIVEN** complete valid manifest evidence has established a Location and its native snapshot namespace +- **WHEN** collection stops after removing the last manifest and before removing old unreferenced objects +- **THEN** its previously synchronized immutable collection record SHALL permit a fresh owned collector to establish the same Location policy without inferring it from orphan names +- **AND** collection SHALL revalidate current retention, complete durable references and exact metadata identity/generation before unlinks +- **AND** missing or malformed provenance, namespace transplants, aliases, changed metadata, unavailable configuration, disabled snapshots and zero retention SHALL preserve remaining bytes +- **AND** every protected durable root SHALL resolve to a valid native manifest before collection; a dangling reference SHALL refuse orphan deletion, while valid references in another Location SHALL remain protected without blocking unrelated orphan reclamation +- **AND** journal/worker fences and retained retry receipts SHALL remain independently required diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 9d9566e0c..9c3eebdf7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -968,3 +968,41 @@ preparation/TUI client cases serially in addition to managed Git recovery; fresh source-bound platform evidence remains required. Partial construction, complete reference/CAS/metadata retirement, default Git admission and M2–M8/parent task acceptance remain open. No public API or SDK schema changes in this increment. + +Final intent-slot CLI evidence: all 22 serial branch/recovery cases pass at +2ec1b1c5 in 837.69s, including both managed Git process-death cases and terminal +lost-reply reconciliation. SDK generation consistency, typecheck and 136 tests +pass at that checkpoint, with ten configured external skips. Native run +42753722365 passes both Linux jobs and macOS ARM; Intel remains running at this +checkpoint. These results do not accept later source changes. + +Collection-provenance continuation (2026-11-10): before any unlink, fallback +collection now publishes/synchronizes an immutable bounded Location/root/native- +namespace record derived from complete valid manifests and pins its exact native +identity/generation through deletion. Restart after the last manifest disappears +can establish fresh retention without guessing from orphan names. Missing legacy +provenance, corrupt/foreign records, aliases, changed metadata, unavailable +configuration and disabled/zero retention protect remaining bytes. Every protected +copy reference must resolve to a valid manifest in the global private inventory +before collection; a dangling root cannot silently become an empty object set. +Valid other-Location roots and unrelated Git IDs retain compatibility. Namespace +metadata and unknown unpublished construction files count toward quota; none gain +cleanup authority from their names. Journal/worker fences remain unchanged. + +Local verification: the snapshot candidate suite passes 205 cases with nine +intentional subprocess/diagnostic skips before the final unavailable-policy and +Git-reference compatibility refinements. Final targeted runs cover all 31 fallback +cases and seven collection unit cases, including the actual process-kill/physical- +reopen proof and its invoked ignored child. Eighteen App reclamation cases pass, +including serialized reference-writer exclusion. The final refinements are confined +to collection and covered by its targeted runs. Focused code/security review caught +the missing-reference and null-configuration cases; both now have regression +coverage. Full metadata/CAS retirement, partial construction and complete M2–M8 +acceptance remain open. Public API/SDK schemas are unchanged; full workspace and +source-bound native/platform acceptance remain required. + +Final collection-provenance checks: all-target snapshot/App/CLI Clippy with warnings +denied, formatting and whitespace pass. Strict OpenSpec passes 100 items; spec lint +reports zero errors and 25 warnings. Native macOS sandbox deprecation and linker +unwind-table diagnostics were observed during builds. No new public/generated +contract changed; the earlier SDK evidence remains scoped to its tested checkpoint. From 50e003721e42e73470ad6a7ea970241a3a7c5498 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 01:00:26 -0400 Subject: [PATCH 62/87] Allow complete native recovery verification on Intel runners --- .github/workflows/non-git-snapshot-proof.yml | 4 +++- docs/goal-delivery.md | 9 +++++++-- openspec/changes/complete-daily-coding-surfaces/tasks.md | 8 ++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index b643ff40b..deae05724 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -40,7 +40,9 @@ jobs: native: name: Snapshot native (${{ matrix.os }}) runs-on: ${{ matrix.os }} - timeout-minutes: 45 + # Intel's complete matrix can exceed 45 minutes while compiling the final + # structured-result binaries after all native recovery checks have passed. + timeout-minutes: 60 strategy: fail-fast: false matrix: diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index b3f58f980..250d4c2cd 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -112,8 +112,13 @@ changes and is not evidence for them. The workflow now also selects the three pre-effect preparation/TUI client cases serially and retains their log; that workflow addition still requires source-bound platform acceptance. Run [42753722365](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42753722365) -at `2ec1b1c5` has passed both Linux jobs and macOS ARM; macOS Intel is still -running at this checkpoint. It predates the collection-provenance continuation. +at `2ec1b1c5` passes both Linux jobs and macOS ARM. Intel is cancelled at the +45-minute job limit while compiling the final structured-result binaries; its +snapshot, core, App, managed Git, preparation/TUI and JSONL steps pass. This is +incomplete platform evidence, not a passing matrix. The workflow job allowance is +now 60 minutes; individual test deadlines and assertions are unchanged. The run +predates the collection-provenance continuation, whose source at `124c11b9` has +its own run 42759174937 in progress. Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 9c3eebdf7..dbd262a3a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1006,3 +1006,11 @@ denied, formatting and whitespace pass. Strict OpenSpec passes 100 items; spec l reports zero errors and 25 warnings. Native macOS sandbox deprecation and linker unwind-table diagnostics were observed during builds. No new public/generated contract changed; the earlier SDK evidence remains scoped to its tested checkpoint. + +Native scheduling correction: run 42753722365 at 2ec1b1c5 finishes both Linux jobs +and macOS ARM successfully. Intel passes snapshot/core/App, managed Git, preparation/ +TUI and JSONL checks, then reaches the 45-minute job limit while compiling final +structured-result binaries; its matrix result is cancelled, not accepted. The job +allowance is now 60 minutes without changing any test deadline/assertion. Collection +source 124c11b9 has its own run 42759174937 in progress and is not accepted from the +prior source's results. From 5be366b7239d9c3adac4052297b8a14283705231 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 02:17:17 -0400 Subject: [PATCH 63/87] Record metadata retirement phases at the Store writer --- Cargo.lock | 1 + crates/cyber-app/Cargo.toml | 1 + crates/cyber-app/src/snapshot_reclamation.rs | 9 + .../src/snapshot_reclamation/metadata.rs | 83 ++++ .../cyber-app/tests/snapshot_reclamation.rs | 47 ++ .../tests/support/snapshot_retirement.rs | 430 ++++++++++++++++++ .../src/runtime/snapshot_reclamation.rs | 36 +- .../runtime/snapshot_reclamation/metadata.rs | 332 ++++++++++++++ crates/cyber-snapshot/src/copy/reclamation.rs | 2 +- crates/cyber-snapshot/src/reclamation.rs | 12 +- ...110000000_snapshot_metadata_retirement.sql | 3 + crates/cyber-store/src/migrate.rs | 37 +- docs/goal-delivery.md | 22 +- .../managed-recovery.md | 28 +- .../specs/snapshots-checkpoints/spec.md | 14 + .../complete-daily-coding-surfaces/tasks.md | 46 ++ 16 files changed, 1082 insertions(+), 21 deletions(-) create mode 100644 crates/cyber-app/src/snapshot_reclamation/metadata.rs create mode 100644 crates/cyber-app/tests/support/snapshot_retirement.rs create mode 100644 crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs create mode 100644 crates/cyber-store/migrations/20261110000000_snapshot_metadata_retirement.sql diff --git a/Cargo.lock b/Cargo.lock index 32eed6515..1da8d909f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -986,6 +986,7 @@ dependencies = [ "aws-lc-rs", "axum", "base64 0.22.1", + "blake3", "cap-std", "chrono", "cyber-account", diff --git a/crates/cyber-app/Cargo.toml b/crates/cyber-app/Cargo.toml index 4148998cd..1d02e98ca 100644 --- a/crates/cyber-app/Cargo.toml +++ b/crates/cyber-app/Cargo.toml @@ -37,6 +37,7 @@ mdns-sd = { version = "0.21", default-features = false } url = "2" [dev-dependencies] +blake3 = "1" aws-lc-rs = "1" jsonwebtoken = { version = "=10.3.0", default-features = false, features = ["aws_lc_rs"] } cyber-client = { path = "../cyber-client" } diff --git a/crates/cyber-app/src/snapshot_reclamation.rs b/crates/cyber-app/src/snapshot_reclamation.rs index dedf9ebd8..10a96afb7 100644 --- a/crates/cyber-app/src/snapshot_reclamation.rs +++ b/crates/cyber-app/src/snapshot_reclamation.rs @@ -1,4 +1,5 @@ //! Compose native cleanup with the same serialized writer that owns references. +mod metadata; use cyber_server::runtime::{ SnapshotOrigin, snapshot_reclamation::{self as records, Completion, Plan, Record}, @@ -110,6 +111,14 @@ impl Policy for Reclamation { .map_err(|error| error.to_string())?; Ok(removed) } + fn retire( + &self, + payload: Plan, + plan: records::metadata::Plan, + delete: Delete, + ) -> Result { + metadata::retire(self, payload, plan, delete) + } fn collect(&self, collect: Collect) -> Result<(), String> { self.0 .transaction(move |db| collect(&records::references(db)?).map_err(invalid)) diff --git a/crates/cyber-app/src/snapshot_reclamation/metadata.rs b/crates/cyber-app/src/snapshot_reclamation/metadata.rs new file mode 100644 index 000000000..1a0c123a3 --- /dev/null +++ b/crates/cyber-app/src/snapshot_reclamation/metadata.rs @@ -0,0 +1,83 @@ +//! Serialize native metadata retirement against the receipt/reference writer. +use super::*; +use records::metadata::{self, Plan as MetadataPlan}; + +pub(super) fn retire( + owner: &Reclamation, + payload: Plan, + plan: MetadataPlan, + delete: Delete, +) -> Result { + plan.validate(&payload).map_err(|error| error.to_string())?; + let existing = owner + .record(&payload.receipt)? + .ok_or("Missing completed payload reclamation")?; + if existing.plan != payload || !existing.completed { + return Err("Metadata retirement payload owner is incomplete or changed".into()); + } + if let Some(retained) = existing.metadata { + if retained != plan { + return Err("Metadata retirement differs from retained request".into()); + } + if existing.retired { + return Ok(0); + } + } else { + let (prepare, original) = (plan.clone(), payload.clone()); + owner + .0 + .append_checked(&plan.payload_plan_id, Expected::Seq(1), move |db| { + prepare.authorize(db, &original)?; + Ok(( + vec![NewEvent::new( + metadata::PLANNED, + serde_json::to_value(prepare).map_err(invalid)?, + )], + (), + )) + }) + .map_err(|error| error.to_string())?; + } + let (consume, original) = (plan.clone(), payload.clone()); + let removed = owner + .0 + .transaction(move |db| { + let record = records::lookup(db, &original.receipt)? + .ok_or_else(|| invalid("Missing metadata retirement owner"))?; + if record.plan != original + || !record.completed + || record.retired + || record.metadata.as_ref() != Some(&consume) + { + return Err(invalid("Metadata retirement changed before deletion")); + } + consume.authorize(db, &original)?; + // Native namespaces, leases and exact file pins belong to this bounded + // callback. It may not perform source work or call back into Store. + let removed = delete().map_err(invalid)?; + if removed != consume.total_bytes()? { + return Err(invalid("Metadata retirement byte accounting changed")); + } + Ok(removed) + }) + .map_err(|error| error.to_string())?; + let id = plan.payload_plan_id.clone(); + owner + .0 + .append_checked(&id, Expected::Seq(2), move |db| { + plan.authorize(db, &payload)?; + let completion = Completion { + plan_id: plan.payload_plan_id.clone(), + removed_bytes: removed, + }; + Ok(( + vec![NewEvent::new( + metadata::COMPLETED, + serde_json::to_value(completion).map_err(invalid)?, + )], + (), + )) + }) + .map_err(|error| error.to_string())?; + Ok(removed) +} diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs index 51b843771..a7326cb73 100644 --- a/crates/cyber-app/tests/snapshot_reclamation.rs +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -1,5 +1,7 @@ #![cfg(any(target_os = "macos", target_os = "linux"))] //! Real App restores/receipts and native private payloads; no source writes by cleanup. +#[path = "support/snapshot_retirement.rs"] +mod retirement; #[path = "support/scripted_model.rs"] mod scripted_model; use cyber_app::{App, AppOptions}; @@ -330,6 +332,9 @@ async fn retention_disabled_capture_disabled_and_live_lifetime_do_not_license_cl let f = Fixture::new().await; assert_eq!(f.reclaim().await.unwrap(), 0); f.age(); + *f.config.lock().unwrap() = Value::Null; + assert_eq!(f.reclaim().await.unwrap(), 0); + assert!(f.record().is_none() && f.journal.join("stage_0").exists()); *f.config.lock().unwrap() = json!({"snapshots":{"retention_days":0}}); assert_eq!(f.reclaim().await.unwrap(), 0); *f.config.lock().unwrap() = json!({"snapshots":false}); @@ -366,6 +371,48 @@ struct Cut { pause: Option, tamper: Option, } + +#[tokio::test] +async fn unavailable_policy_after_one_unlink_preserves_remaining_payloads_and_resumes() { + let f = Fixture::new().await; + f.age(); + let first = f.journal.join("stage_0"); + let remaining = f.journal.join("stage_1"); + let bytes = std::fs::read(&remaining).unwrap(); + let inference = f.model.requests().len(); + let cut = GitSnapshots::with_reclamation( + f.paths.data.clone(), + Arc::new(move |_| { + if first.exists() { + json!({}) + } else { + Value::Null + } + }), + vec![ + f.paths.config.clone(), + f.paths.state.clone(), + f.paths.cache.clone(), + f.paths.tmp.clone(), + ], + None, + Some(cyber_app::snapshot_reclamation::policy(f.app.store.clone())), + ); + assert!( + cut.reclaim(f.project.to_str().unwrap()) + .await + .unwrap_err() + .contains("retention") + ); + assert!(!f.journal.join("stage_0").exists()); + assert_eq!(std::fs::read(&remaining).unwrap(), bytes); + assert!(!f.record().unwrap().completed); + assert!(f.reclaim().await.unwrap() > 0); + assert!(!remaining.exists()); + assert!(f.record().unwrap().completed); + assert_eq!(f.model.requests().len(), inference); + f.app.runtime.shutdown().await; +} impl Policy for Cut { fn receipt( &self, diff --git a/crates/cyber-app/tests/support/snapshot_retirement.rs b/crates/cyber-app/tests/support/snapshot_retirement.rs new file mode 100644 index 000000000..22d87d3db --- /dev/null +++ b/crates/cyber-app/tests/support/snapshot_retirement.rs @@ -0,0 +1,430 @@ +//! Writer protocol only: events do not perform or prove native metadata deletion. +use super::*; +use cyber_server::runtime::snapshot_reclamation::metadata::{self, File, Plan, Role}; +use cyber_store::{Expected, NewEvent}; +use std::os::unix::fs::MetadataExt; + +fn plan(f: &Fixture) -> Plan { + let lifetime = &f.receipt.checkout.as_ref().unwrap().lifetime; + let workers = f + .journal + .parent() + .unwrap() + .parent() + .unwrap() + .join("workers"); + let files = [ + (Role::Intent, f.journal.join("intent.json")), + (Role::Spare, f.journal.join("intent.next")), + (Role::Lifetime, workers.join(&lifetime.owner.activity_id)), + ( + Role::SourcePhase, + workers.join(lifetime.source_phase.as_ref().unwrap()), + ), + ] + .into_iter() + .map(|(role, path)| { + let metadata = std::fs::metadata(&path).unwrap(); + let birth = metadata + .created() + .unwrap() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + let generation = serde_json::to_vec(&( + metadata.dev(), + metadata.ino(), + metadata.mode(), + metadata.uid(), + metadata.gid(), + metadata.nlink(), + metadata.len(), + metadata.mtime(), + metadata.mtime_nsec(), + metadata.ctime(), + metadata.ctime_nsec(), + )) + .unwrap(); + File { + owner: lifetime.owner.activity_id.clone(), + role, + identity: format!( + "{}:{}:{}:{}", + metadata.dev(), + metadata.ino(), + birth.as_secs(), + birth.subsec_nanos() + ), + generation: blake3::hash(&generation).to_hex().to_string(), + hash: blake3::hash(&std::fs::read(path).unwrap()) + .to_hex() + .to_string(), + bytes: metadata.len(), + } + }) + .collect(); + Plan { + version: 1, + payload_plan_id: records::Plan::id(&f.receipt).unwrap(), + files, + } +} +fn append( + f: &Fixture, + sequence: i64, + kind: &str, + value: Value, +) -> Result<(), cyber_store::StoreError> { + f.app + .store + .append( + &records::Plan::id(&f.receipt).unwrap(), + Expected::Seq(sequence), + vec![NewEvent::new(kind, value)], + ) + .map(|_| ()) +} +fn completed(plan: &Plan) -> Value { + json!(records::Completion { + plan_id: plan.payload_plan_id.clone(), + removed_bytes: plan.total_bytes().unwrap() + }) +} +async fn payload_complete(f: &Fixture) -> Plan { + f.age(); + assert!(f.reclaim().await.unwrap() > 0); + let plan = plan(f); + plan.validate(&f.record().unwrap().plan).unwrap(); + plan +} + +#[tokio::test] +async fn metadata_history_distinguishes_phases_and_reopens_without_native_effects() { + let f = Fixture::new().await; + let plan = payload_complete(&f).await; + let intent = std::fs::read(f.journal.join("intent.json")).unwrap(); + let requests = f.model.requests().len(); + append(&f, 1, metadata::PLANNED, json!(plan)).unwrap(); + let record = f.record().unwrap(); + assert!(record.completed && !record.retired); + assert_eq!(record.metadata, Some(plan.clone())); + // The trusted writer is explicitly supplying a completion in this Store + // protocol test. Native retirement must separately validate its deletion + // callback; accepting these events is not native deletion acceptance. + append(&f, 2, metadata::COMPLETED, completed(&plan)).unwrap(); + assert!(f.record().unwrap().retired); + let receipt = f.receipt.clone(); + f.app.runtime.shutdown().await; + let reopened = cyber_store::Store::open(cyber_store::StoreOptions::new( + DatabaseLocation::File(f.paths.data.join("cyber.db")), + cyber_server::runtime::Runtime::registry(), + )) + .unwrap(); + let record = reopened + .read(move |db| records::lookup(db, &receipt)) + .unwrap() + .unwrap(); + assert!(record.completed && record.retired); + assert_eq!(record.metadata, Some(plan)); + assert_eq!( + std::fs::read(f.journal.join("intent.json")).unwrap(), + intent + ); + assert_eq!(f.model.requests().len(), requests); +} + +#[tokio::test] +async fn metadata_retirement_cannot_precede_payload_completion() { + let f = Fixture::new().await; + f.age(); + planned_without_deletion(&f).await; + let plan = plan(&f); + let notifications = f.app.store.subscribe(); + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + assert!( + policy + .retire( + f.record().unwrap().plan, + plan.clone(), + Box::new(|| panic!("incomplete payload invoked retirement")) + ) + .is_err() + ); + assert!(append(&f, 0, metadata::PLANNED, json!(plan)).is_err()); + assert!(!f.record().unwrap().completed); + assert!(f.record().unwrap().metadata.is_none()); + assert!(f.journal.join("stage_0").exists()); + assert!(notifications.try_recv().is_err()); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn metadata_plan_rejects_foreign_missing_duplicate_and_unbounded_evidence() { + let f = Fixture::new().await; + let valid = payload_complete(&f).await; + let notifications = f.app.store.subscribe(); + for mode in [ + "version", + "plan", + "owner", + "owner_size", + "missing", + "duplicate", + "array_size", + "identity", + "alias", + "cancellation", + "hash", + "size", + "path", + ] { + let mut value = json!(valid); + corrupt(&mut value, mode); + assert!(append(&f, 1, metadata::PLANNED, value).is_err(), "{mode}"); + assert!(f.record().unwrap().metadata.is_none(), "{mode}"); + } + assert!(notifications.try_recv().is_err()); + assert!(f.journal.join("intent.json").exists() && f.journal.join("intent.next").exists()); + f.app.runtime.shutdown().await; +} +fn corrupt(plan: &mut Value, mode: &str) { + match mode { + "version" => plan["version"] = json!(2), + "plan" => plan["payload_plan_id"] = json!(format!("snapshot_reclaim_{}", "f".repeat(64))), + "owner" => plan["files"][0]["owner"] = json!("op_foreign"), + "owner_size" => plan["files"][0]["owner"] = json!("x".repeat(128 * 1024)), + "missing" => { + plan["files"].as_array_mut().unwrap().remove(2); + } + "duplicate" => { + let file = plan["files"][0].clone(); + plan["files"].as_array_mut().unwrap().push(file); + } + "array_size" => plan["files"] = json!(vec![plan["files"][0].clone(); 11]), + "identity" => plan["files"][2]["identity"] = json!("1:2:3:4"), + "alias" => plan["files"][1]["identity"] = plan["files"][0]["identity"].clone(), + "cancellation" => plan["files"][1]["role"] = json!("cancellation"), + "hash" => plan["files"][0]["hash"] = json!("not a digest"), + "size" => plan["files"][0]["bytes"] = json!(u64::MAX), + "path" => plan["files"][0]["path"] = json!("../../owner"), + _ => panic!("unknown corruption"), + } +} + +#[tokio::test] +async fn metadata_plan_is_immutable_and_completion_rechecks_later_native_fences() { + let f = Fixture::new().await; + let plan = payload_complete(&f).await; + append(&f, 1, metadata::PLANNED, json!(plan)).unwrap(); + let payload = f.record().unwrap().plan; + let mut rewritten = payload.clone(); + rewritten.files[0].identity = "1:2:3:4".into(); + let same_receipt = plan.clone(); + assert!( + f.app + .store + .read(move |db| same_receipt.authorize(db, &rewritten)) + .is_err() + ); + let mut changed = plan.clone(); + changed.files[0].generation = "0".repeat(64); + assert!( + f.app + .store + .read(move |db| changed.authorize(db, &payload)) + .is_err() + ); + let mut completion = completed(&plan); + completion["removed_bytes"] = json!(plan.total_bytes().unwrap() + 1); + assert!(append(&f, 2, metadata::COMPLETED, completion).is_err()); + f.app + .store + .append( + "op_later_unknown", + Expected::Seq(-1), + vec![NewEvent::new( + "native.activity.changed.1", + json!({"session_id":f.session,"status":"unknown","phase":"legacy_process"}), + )], + ) + .unwrap(); + assert!(append(&f, 2, metadata::COMPLETED, completed(&plan)).is_err()); + assert!(!f.record().unwrap().retired); + assert_eq!( + f.app + .store + .reproject( + &[], + "snapshot.reclamation.", + Arc::new(|db, event| records::project(db, event)) + ) + .unwrap(), + 3 + ); + assert!(f.record().unwrap().metadata.is_some()); + assert!(f.journal.join("intent.json").exists()); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn completed_metadata_history_reprojects_after_session_deletion() { + let f = Fixture::new().await; + let plan = payload_complete(&f).await; + append(&f, 1, metadata::PLANNED, json!(plan)).unwrap(); + append(&f, 2, metadata::COMPLETED, completed(&plan)).unwrap(); + f.app.runtime.delete(&f.session).await.unwrap(); + assert_eq!( + f.app + .store + .reproject( + &[], + "snapshot.reclamation.", + Arc::new(|db, event| records::project(db, event)) + ) + .unwrap(), + 4 + ); + assert!(f.record().unwrap().retired); + assert!(append(&f, 3, metadata::COMPLETED, completed(&plan)).is_err()); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn metadata_writer_retains_the_plan_and_never_replays_a_completed_callback() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let f = Fixture::new().await; + let plan = payload_complete(&f).await; + let payload = f.record().unwrap().plan; + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + let calls = Arc::new(AtomicUsize::new(0)); + let count = calls.clone(); + assert!( + policy + .retire( + payload.clone(), + plan.clone(), + Box::new(move || { + count.fetch_add(1, Ordering::SeqCst); + Err("interrupted native callback".into()) + }) + ) + .is_err() + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(f.record().unwrap().metadata, Some(plan.clone())); + assert!(!f.record().unwrap().retired); + let mut changed = plan.clone(); + changed.files[0].generation = "0".repeat(64); + assert!( + policy + .retire( + payload.clone(), + changed, + Box::new(|| panic!("changed plan invoked native work")) + ) + .is_err() + ); + let total = plan.total_bytes().unwrap(); + let count = calls.clone(); + assert!( + policy + .retire( + payload.clone(), + plan.clone(), + Box::new(move || { + count.fetch_add(1, Ordering::SeqCst); + Ok(total + 1) + }) + ) + .is_err() + ); + assert!(!f.record().unwrap().retired); + let count = calls.clone(); + assert_eq!( + policy + .retire( + payload.clone(), + plan.clone(), + Box::new(move || { + count.fetch_add(1, Ordering::SeqCst); + Ok(total) + }) + ) + .unwrap(), + total + ); + assert!(f.record().unwrap().retired); + assert_eq!( + policy + .retire( + payload, + plan, + Box::new(|| panic!("retired callback replayed")) + ) + .unwrap(), + 0 + ); + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert!(f.journal.join("intent.json").exists()); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn metadata_callback_excludes_the_reference_writer() { + let f = Fixture::new().await; + let plan = payload_complete(&f).await; + let payload = f.record().unwrap().plan; + let total = plan.total_bytes().unwrap(); + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + let (started, entered) = std::sync::mpsc::channel(); + let (release, gate) = std::sync::mpsc::channel(); + let retiring = std::thread::spawn(move || { + policy.retire( + payload, + plan, + Box::new(move || { + started.send(()).unwrap(); + gate.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(total) + }), + ) + }); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + let wrote = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (store, flag) = (f.app.store.clone(), wrote.clone()); + let writer = std::thread::spawn(move || { + store.transaction(move |_| { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }) + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!wrote.load(std::sync::atomic::Ordering::SeqCst)); + release.send(()).unwrap(); + assert_eq!(retiring.join().unwrap().unwrap(), total); + writer.join().unwrap().unwrap(); + assert!(f.record().unwrap().retired); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn legacy_maintenance_policy_refuses_metadata_work_without_invoking_the_callback() { + let f = Fixture::new().await; + let plan = payload_complete(&f).await; + let unavailable = Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after_delete: false, + pause: None, + tamper: None, + }; + let error = unavailable + .retire( + f.record().unwrap().plan, + plan, + Box::new(|| panic!("unsupported retirement invoked native work")), + ) + .unwrap_err(); + assert!(error.contains("unavailable")); + assert!(f.record().unwrap().metadata.is_none()); + assert!(f.journal.join("intent.json").exists()); + f.app.runtime.shutdown().await; +} diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation.rs b/crates/cyber-server/src/runtime/snapshot_reclamation.rs index 5aea49e21..a6af55efa 100644 --- a/crates/cyber-server/src/runtime/snapshot_reclamation.rs +++ b/crates/cyber-server/src/runtime/snapshot_reclamation.rs @@ -1,4 +1,5 @@ //! Durable private-payload cleanup plans. These never settle native activities. +pub mod metadata; use super::{ SnapshotLifetimeBinding, SnapshotOrigin, SnapshotPreparationReceipt, SnapshotRecoveryReceipt, SnapshotRestoreReceipt, activity, @@ -149,7 +150,12 @@ pub struct Completion { #[derive(Debug, Clone)] pub struct Record { pub plan: Plan, + /// Observed payload-completion event; metadata may still be retained. pub completed: bool, + /// The immutable metadata retirement request, if its writer observed it. + pub metadata: Option, + /// Observed metadata-completion event, never a standalone native capability. + pub retired: bool, } fn invalid(message: impl Into) -> StoreError { StoreError::InvalidAppend(message.into()) @@ -329,27 +335,22 @@ pub fn lookup>( if kind != PLANNED || plan.receipt != receipt { return Err(invalid("Reclamation request owner differs")); } - let (seq, kind, raw): (i64, String, String) = db.query_row( - "SELECT seq,type,data FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", + let (seq, kind): (i64, String) = db.query_row( + "SELECT seq,type FROM event WHERE aggregate_id=?1 ORDER BY seq DESC LIMIT 1", [&id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + |row| Ok((row.get(0)?, row.get(1)?)), )?; - let completed = match seq { - 0 if kind == PLANNED => false, - 1 if kind == COMPLETED => { - let completion: Completion = - serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; - if completion.plan_id != id || completion.removed_bytes != plan.total_bytes()? { - return Err(invalid("Reclamation completion differs")); - } - true - } - _ => return Err(invalid("Invalid reclamation history")), - }; - Ok(Some(Record { plan, completed })) + let (completed, metadata, retired) = metadata::history(db, &id, &plan, seq, &kind)?; + Ok(Some(Record { + plan, + completed, + metadata, + retired, + })) } pub(super) fn register(registry: &mut EventRegistry) { + metadata::register(registry); registry .register_with(PLANNED, |value| { serde_json::from_value::(value.clone()) @@ -369,6 +370,9 @@ pub(super) fn register(registry: &mut EventRegistry) { /// Validate append ownership while allowing historical projection rebuilds after /// later fences or Session deletion. Replaying records never grants file effects. pub fn project(db: &Connection, event: &StoredEvent) -> Result<(), String> { + if matches!(event.kind.as_str(), metadata::PLANNED | metadata::COMPLETED) { + return metadata::project(db, event); + } let current = if matches!(event.kind.as_str(), PLANNED | COMPLETED) { db.query_row( "SELECT seq FROM event_sequence WHERE aggregate_id=?1", diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs b/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs new file mode 100644 index 000000000..53db951b1 --- /dev/null +++ b/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs @@ -0,0 +1,332 @@ +//! Receipt-bound metadata retirement history; shape validation grants no native I/O. +use super::{Completion, Plan as PayloadPlan, Receipt, invalid}; +use cyber_store::{EventRegistry, StoreError, StoredEvent}; +use rusqlite::{Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +pub const PLANNED: &str = "snapshot.reclamation.metadata.planned.1"; +pub const COMPLETED: &str = "snapshot.reclamation.metadata.completed.1"; +const FILE_LIMIT: usize = 10; +const FILE_BYTES: u64 = 8 * 1024 * 1024; +const ENCODED_LIMIT: usize = 16 * 1024; + +struct EncodingLimit(usize); +impl std::io::Write for EncodingLimit { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 = self + .0 + .checked_add(bytes.len()) + .filter(|size| *size <= ENCODED_LIMIT) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "metadata encoding exceeds limit", + ) + })?; + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +fn bounded_encoding(value: &impl Serialize) -> Result<(), String> { + serde_json::to_writer(EncodingLimit(0), value) + .map_err(|_| "Metadata retirement encoding exceeds limit".into()) +} + +/// The native adapter derives each route from the receipt and role, never from +/// a caller-provided pathname. These are unique regular metadata files only. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Role { + Intent, + Spare, + Cancellation, + Lifetime, + SourcePhase, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct File { + pub owner: String, + pub role: Role, + pub identity: String, + pub generation: String, + pub hash: String, + pub bytes: u64, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Plan { + pub version: u32, + pub payload_plan_id: String, + pub files: Vec, +} +impl Plan { + fn shape(&self) -> Result<(), StoreError> { + if self.version != 1 + || self.files.is_empty() + || self.files.len() > FILE_LIMIT + || !self + .payload_plan_id + .strip_prefix("snapshot_reclaim_") + .is_some_and(super::hash) + { + return Err(invalid("Invalid metadata retirement plan bounds")); + } + let mut roles = BTreeSet::new(); + let mut identities = BTreeSet::new(); + for file in &self.files { + if file.owner.is_empty() + || file.owner.len() > 128 + || !file + .owner + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + || !native_identity(&file.identity) + || !super::hash(&file.generation) + || !super::hash(&file.hash) + || file.bytes > FILE_BYTES + || !roles.insert((&file.owner, file.role)) + || !identities.insert(&file.identity) + { + return Err(invalid( + "Invalid metadata retirement file or duplicate role", + )); + } + } + // Bound all strings before allocating their serialized representation. + bounded_encoding(self).map_err(invalid)?; + self.total_bytes().map(drop) + } + pub fn validate(&self, payload: &PayloadPlan) -> Result<(), StoreError> { + self.shape()?; + payload.validate()?; + if self.payload_plan_id != PayloadPlan::id(&payload.receipt)? { + return Err(invalid("Metadata retirement has a foreign payload plan")); + } + let owners = payload.receipt.owners()?; + for owner in &owners { + if owner + .lifetime + .checkout_resources_digest(&[]) + .map_err(invalid)? + != owner.resources_digest + { + return Err(invalid( + "Metadata retirement needs an empty complete checkout inventory", + )); + } + for role in [Role::Intent, Role::Lifetime, Role::SourcePhase] { + if !self + .files + .iter() + .any(|file| file.owner == owner.origin.activity_id && file.role == role) + { + return Err(invalid("Metadata retirement is missing a worker role")); + } + } + } + for file in &self.files { + let owner = owners + .iter() + .find(|owner| owner.origin.activity_id == file.owner) + .ok_or_else(|| invalid("Metadata retirement has a foreign worker"))?; + if file.role == Role::Lifetime && file.identity != owner.lifetime.file_identity { + return Err(invalid("Metadata retirement lifetime identity differs")); + } + if file.role == Role::Cancellation + && !matches!(payload.receipt, Receipt::Preparation(_)) + { + return Err(invalid("Unexpected cancellation metadata retirement role")); + } + } + if matches!(payload.receipt, Receipt::Preparation(_)) + && !self + .files + .iter() + .any(|file| file.role == Role::Cancellation) + { + return Err(invalid("Missing cancellation metadata retirement role")); + } + Ok(()) + } + pub fn total_bytes(&self) -> Result { + self.files.iter().try_fold(0u64, |sum, file| { + sum.checked_add(file.bytes) + .ok_or_else(|| invalid("Metadata retirement size overflow")) + }) + } + /// Re-attest at the writer around a separately verified native callback. + /// This method never proves inode ownership or performs filesystem effects. + pub fn authorize( + &self, + db: &Connection, + payload: &PayloadPlan, + ) -> Result, StoreError> { + self.validate(payload)?; + if read_payload(db, &self.payload_plan_id)? != *payload { + return Err(invalid("Metadata retirement payload inventory changed")); + } + payload_completed(db, &self.payload_plan_id, payload)?; + let retained: Option<(String, String)> = db + .query_row( + "SELECT type,data FROM event WHERE aggregate_id=?1 AND seq=2", + [&self.payload_plan_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + if let Some((kind, raw)) = retained { + let plan: Plan = serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; + if kind != PLANNED || plan != *self { + return Err(invalid( + "Metadata retirement differs from the retained plan", + )); + } + } + payload.authorize(db) + } +} +fn native_identity(value: &str) -> bool { + if value.len() > 128 { + return false; + } + let parts = value.split(':').collect::>(); + value.len() <= 128 + && parts.len() == 4 + && parts.iter().all(|part| { + !part.is_empty() + && part.bytes().all(|byte| byte.is_ascii_digit()) + && part.parse::().is_ok() + }) + && parts[3] + .parse::() + .is_ok_and(|nanos| nanos < 1_000_000_000) +} +fn read_payload(db: &Connection, id: &str) -> Result { + let raw = event_at(db, id, 0, super::PLANNED)?; + let payload: PayloadPlan = serde_json::from_str(&raw).map_err(|e| invalid(e.to_string()))?; + payload.validate()?; + if PayloadPlan::id(&payload.receipt)? != id { + return Err(invalid("Foreign metadata retirement aggregate")); + } + Ok(payload) +} +fn event_at(db: &Connection, id: &str, sequence: i64, kind: &str) -> Result { + db.query_row( + "SELECT data FROM event WHERE aggregate_id=?1 AND seq=?2 AND type=?3", + rusqlite::params![id, sequence, kind], + |row| row.get(0), + ) + .map_err(StoreError::from) +} +fn check_completion(raw: &str, id: &str, bytes: u64) -> Result<(), StoreError> { + let completed: Completion = serde_json::from_str(raw).map_err(|e| invalid(e.to_string()))?; + if completed.plan_id != id || completed.removed_bytes != bytes { + return Err(invalid("Metadata or payload retirement completion differs")); + } + Ok(()) +} +fn payload_completed(db: &Connection, id: &str, payload: &PayloadPlan) -> Result<(), StoreError> { + check_completion( + &event_at(db, id, 1, super::COMPLETED)?, + id, + payload.total_bytes()?, + ) +} +fn planned(db: &Connection, id: &str, payload: &PayloadPlan) -> Result { + let plan: Plan = + serde_json::from_str(&event_at(db, id, 2, PLANNED)?).map_err(|e| invalid(e.to_string()))?; + plan.validate(payload)?; + Ok(plan) +} + +pub(super) fn history( + db: &Connection, + id: &str, + payload: &PayloadPlan, + sequence: i64, + kind: &str, +) -> Result<(bool, Option, bool), StoreError> { + match (sequence, kind) { + (0, super::PLANNED) => Ok((false, None, false)), + (1, super::COMPLETED) => { + payload_completed(db, id, payload)?; + Ok((true, None, false)) + } + (2, PLANNED) | (3, COMPLETED) => { + payload_completed(db, id, payload)?; + let plan = planned(db, id, payload)?; + if sequence == 3 { + check_completion(&event_at(db, id, 3, COMPLETED)?, id, plan.total_bytes()?)?; + } + Ok((true, Some(plan), sequence == 3)) + } + _ => Err(invalid( + "Invalid reclamation or metadata retirement history", + )), + } +} + +pub(super) fn register(registry: &mut EventRegistry) { + registry + .register_with(PLANNED, |value| { + // Reject oversized vectors/encoding before cloning or deserializing + // the event's JSON into owned metadata fields. + if value + .get("files") + .and_then(serde_json::Value::as_array) + .is_none_or(|files| files.is_empty() || files.len() > FILE_LIMIT) + { + return Err("Invalid metadata retirement file bounds".into()); + } + bounded_encoding(value)?; + serde_json::from_value::(value.clone()) + .map_err(|e| e.to_string())? + .shape() + .map_err(|e| e.to_string()) + }) + .expect("metadata retirement plan event"); + registry + .register_with(COMPLETED, |value| { + serde_json::from_value::(value.clone()) + .map(drop) + .map_err(|e| e.to_string()) + }) + .expect("metadata retirement completion event"); +} +pub(super) fn project(db: &Connection, event: &StoredEvent) -> Result<(), String> { + let payload = read_payload(db, &event.aggregate_id).map_err(|e| e.to_string())?; + payload_completed(db, &event.aggregate_id, &payload).map_err(|e| e.to_string())?; + let plan = match event.kind.as_str() { + PLANNED if event.seq == 2 => { + serde_json::from_value::(event.data.clone()).map_err(|e| e.to_string())? + } + COMPLETED if event.seq == 3 => { + let plan = planned(db, &event.aggregate_id, &payload).map_err(|e| e.to_string())?; + check_completion( + &event.data.to_string(), + &event.aggregate_id, + plan.total_bytes().map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + plan + } + _ => return Err("Invalid metadata retirement event sequence".into()), + }; + plan.validate(&payload).map_err(|e| e.to_string())?; + let current: i64 = db + .query_row( + "SELECT seq FROM event_sequence WHERE aggregate_id=?1", + [&event.aggregate_id], + |row| row.get(0), + ) + .optional() + .map_err(|e| e.to_string())? + .unwrap_or(-1); + if current < event.seq { + plan.authorize(db, &payload).map_err(|e| e.to_string())?; + } + Ok(()) +} diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index 9c53713f2..b0a2f1a2b 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -72,7 +72,7 @@ impl MetadataPin { } fn due(config: &crate::ConfigFn, root: &Path, settled_at_ms: i64) -> bool { let value = config(root); - if value.get("snapshots") == Some(&serde_json::Value::Bool(false)) { + if !value.is_object() || value.get("snapshots") == Some(&serde_json::Value::Bool(false)) { return false; } let days = value diff --git a/crates/cyber-snapshot/src/reclamation.rs b/crates/cyber-snapshot/src/reclamation.rs index 5c7034159..e924db692 100644 --- a/crates/cyber-snapshot/src/reclamation.rs +++ b/crates/cyber-snapshot/src/reclamation.rs @@ -15,11 +15,21 @@ pub type Delete = Box Result + Send>; pub type Collect = Box) -> Result<(), String> + Send>; /// Implementations serialize verification/deletion against reference writers. /// A durable plan must be committed before invoking `delete`; native guards are -/// retained by that closure. Neither method may acknowledge a native activity. +/// retained by that closure. Maintenance never acknowledges a native activity. pub trait Policy: Send + Sync { fn receipt(&self, origin: &SnapshotOrigin) -> Result, String>; fn record(&self, receipt: &Receipt) -> Result, String>; fn reclaim(&self, plan: Plan, delete: Delete) -> Result; + /// Retire exact metadata only after payload completion. The callback must + /// retain native ownership proof; the default never invokes it. + fn retire( + &self, + _payload: Plan, + _plan: cyber_server::runtime::snapshot_reclamation::metadata::Plan, + _delete: Delete, + ) -> Result { + Err("Snapshot metadata retirement is unavailable".into()) + } fn collect(&self, collect: Collect) -> Result<(), String>; fn refused(&self, directory: &str, error: &str); } diff --git a/crates/cyber-store/migrations/20261110000000_snapshot_metadata_retirement.sql b/crates/cyber-store/migrations/20261110000000_snapshot_metadata_retirement.sql new file mode 100644 index 000000000..c43c610fa --- /dev/null +++ b/crates/cyber-store/migrations/20261110000000_snapshot_metadata_retirement.sql @@ -0,0 +1,3 @@ +-- Metadata retirement adds immutable planned/completed phases to reclamation history. +-- Earlier writers must not ignore these phases or reuse their native evidence. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index 7ec29ee18..2b12201bd 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -205,6 +205,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261107030000_snapshot_reference_index", sql: include_str!("../migrations/20261107030000_snapshot_reference_index.sql"), }, + Migration { + id: "20261110000000_snapshot_metadata_retirement", + sql: include_str!("../migrations/20261110000000_snapshot_metadata_retirement.sql"), + }, ]; pub(crate) enum Outcome { @@ -584,7 +588,7 @@ mod tests { "INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('old','ses_old',0,'retained.1',?1,1)", [&evidence], ).unwrap(); - migrate(&mut db, MIGRATIONS).unwrap(); + migrate(&mut db, &MIGRATIONS[..=at]).unwrap(); let roots = |db: &Connection| -> Vec<(String, String)> { db.prepare("SELECT event_id,root FROM snapshot_reference ORDER BY event_id,root") .unwrap() @@ -649,6 +653,37 @@ mod tests { assert_eq!(unknown, [MIGRATIONS[at].id]); } + #[test] + fn metadata_retirement_catalog_survives_reopen_and_preserves_unknown_evidence() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("retirement.db"); + let marker = temp.path().join("retained-intent.next"); + std::fs::write(&marker, b"unknown native metadata").unwrap(); + let at = MIGRATIONS + .iter() + .position(|migration| migration.id == "20261110000000_snapshot_metadata_retirement") + .unwrap(); + let mut db = Connection::open(&path).unwrap(); + migrate(&mut db, &MIGRATIONS[..at]).unwrap(); + let evidence = r#"{"retained":"unknown native owner","metadata_plan":"exact bytes"}"#; + db.execute("INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES ('evt_meta','snapshot_reclaim_fixture',2,'snapshot.reclamation.metadata.planned.1',?1,1)", [evidence]).unwrap(); + migrate(&mut db, &MIGRATIONS[..=at]).unwrap(); + drop(db); + let mut db = Connection::open(path).unwrap(); + let Outcome::TooNew(unknown) = migrate(&mut db, &MIGRATIONS[..at]).unwrap() else { + panic!("previous writer accepted metadata retirement storage"); + }; + assert_eq!(unknown, [MIGRATIONS[at].id]); + assert_eq!( + db.query_row("SELECT data FROM event WHERE id='evt_meta'", [], |row| { + row.get::<_, String>(0) + }) + .unwrap(), + evidence + ); + assert_eq!(std::fs::read(marker).unwrap(), b"unknown native metadata"); + } + #[test] fn migrating_twice_is_a_no_op() { let mut conn = Connection::open_in_memory().unwrap(); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 250d4c2cd..8a4283638 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -105,6 +105,24 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. restart, alias, namespace-transplant and reference regressions pass locally. This does not retire journal/worker/checkout metadata or complete M6. +8. **Metadata retirement now has a durable writer protocol, not native deletion.** + Reclamation history distinguishes payload completion, an immutable metadata + role/identity/generation/hash plan, and metadata completion. The App maintenance + port commits the plan first, excludes reference writers during its guarded + callback, checks byte totals and never repeats an observed completed callback. + Migration catalog fencing preserves older evidence; logical history survives + Store reopen and reprojection after later native fences or Session deletion. + The Rust maintenance port defaults to unavailable for older implementations. + No native producer invokes it yet: ordinary cleanup still retains journal, + phase and lifetime files. Native deletion/resumption, receipt reconciliation, + managed claims and partial-construction acceptance remain required by M6. + + A separate real App regression reproduced 57,344 bytes of payload deletion + when configuration resolved to null. Cleanup now preserves bytes initially and + stops after policy loss between unlinks, retaining its exact resumable plan. + Twenty-seven App cases and 14 migration tests pass locally; native deletion is + not inferred from the writer callback tests. + Native run [42733865471](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42733865471) passes Linux x64/ARM and macOS ARM/Intel at `e2b3ba18`, including the earlier managed Git cleanup and structured CLI cases. It predates the owned intent-slot @@ -118,7 +136,9 @@ snapshot, core, App, managed Git, preparation/TUI and JSONL steps pass. This is incomplete platform evidence, not a passing matrix. The workflow job allowance is now 60 minutes; individual test deadlines and assertions are unchanged. The run predates the collection-provenance continuation, whose source at `124c11b9` has -its own run 42759174937 in progress. +its own run [42759174937](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42759174937), +now successful on all four Linux/macOS platforms. It predates the metadata writer +protocol and payload-policy fix; fresh source-bound acceptance remains required. Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 0f388d19b..f331493ee 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -946,7 +946,8 @@ bytes and the original plan identity for idempotent retry. The plan lives in the Store, so a full snapshot quota cannot prevent recording cleanup eligibility. Retention uses the durable settlement time, with a fresh policy check before each -unlink. Disabled snapshots and zero retention preserve payloads. Cap pressure may +unlink. Unavailable configuration, disabled snapshots and zero retention preserve +payloads; failed configuration resolution cannot select default retention. Cap pressure may attempt this same eligible cleanup and retry capture once; it never overrides reference, lifetime, acknowledgement or retention checks. Periodic fallback-object collection also takes its reference view at the serialized Store writer, rather @@ -961,6 +962,31 @@ The same complete index protects staged, inactive-branch and saved-version histo A missing/unreadable/oversized index refuses cleanup. Reference lookup at the writer must not repeatedly parse the entire conversation history while other writes wait. +### Durable metadata retirement + +After observed payload-cleanup completion, metadata retirement SHALL have separate +planned/completed events at sequences 2/3 of the same receipt-bound reclamation +aggregate. The bounded plan SHALL bind each consumed worker and semantic metadata +role (intent, spare, cancellation, lifetime or source phase) to its exact native +identity, metadata generation, content hash and byte count. Filenames and ambient +paths SHALL NOT be caller-supplied metadata capabilities. Unknown/duplicate roles, +foreign owners, changed plan IDs and inconsistent byte totals SHALL refuse. + +Planning and completion SHALL re-attest the original settlement, complete durable +references and independent native-owner fences at the Store writer. Older writers +SHALL detect the new event protocol through the migration catalog and refuse +mutation. Historical projection rebuilds MAY validate the immutable event history +after later fences or Session deletion without granting any filesystem capability. +The retained Store receipt SHALL distinguish payload completion, pending metadata +retirement and observed retirement completion, including after physical Store reopen. + +Event validation alone SHALL NOT authorize native deletion or establish that it +occurred. The native adapter must independently retain the exact namespace, +reader/lifetime/checkout proof and per-leaf pins around its bounded deletion +callback. The first writer frame supports empty complete checkout inventories; +managed checkout claim retirement, partial constructors and all actual native +retirement/reconciliation/crash paths remain required before M6 acceptance. + ### Paired recovery payload cleanup Acknowledged keep/rollback cleanup SHALL bind the exact durable two-owner recovery diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index e6e1a99f0..fd4912aa5 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -520,3 +520,17 @@ - **AND** missing or malformed provenance, namespace transplants, aliases, changed metadata, unavailable configuration, disabled snapshots and zero retention SHALL preserve remaining bytes - **AND** every protected durable root SHALL resolve to a valid native manifest before collection; a dangling reference SHALL refuse orphan deletion, while valid references in another Location SHALL remain protected without blocking unrelated orphan reclamation - **AND** journal/worker fences and retained retry receipts SHALL remain independently required + +#### Scenario: Payload cleanup loses configuration after its first unlink +- **GIVEN** exact native ownership and a durable payload cleanup plan permit reclamation +- **WHEN** configuration becomes unavailable before a later private unlink +- **THEN** cleanup SHALL preserve the remaining payloads and leave its durable plan incomplete +- **AND** a fresh eligible retry SHALL reconcile that exact plan without source changes or inference + +#### Scenario: Store retains metadata retirement phases independently of payload completion +- **GIVEN** an exact receipt-bound payload plan has observed durable completion +- **WHEN** its metadata retirement writer commits a bounded semantic-role plan +- **THEN** the Store SHALL distinguish pending metadata retirement from payload completion and observed metadata completion after physical reopen +- **AND** foreign or duplicate worker roles, supplied paths, changed plan identities, missing payload completion and inconsistent byte counts SHALL refuse +- **AND** later native-owner fences and incomplete reference views SHALL refuse new retirement writes; historical projection rebuilds SHALL preserve immutable history without authorizing native I/O +- **AND** accepting the event schema alone SHALL NOT prove native metadata deletion or relax independent checkout, namespace, lifetime, reader or retention checks diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index dbd262a3a..94d475752 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1014,3 +1014,49 @@ structured-result binaries; its matrix result is cancelled, not accepted. The jo allowance is now 60 minutes without changing any test deadline/assertion. Collection source 124c11b9 has its own run 42759174937 in progress and is not accepted from the prior source's results. + +Metadata-retirement writer continuation (2026-11-10): reclamation sequences 2/3 +now retain a bounded semantic-role native identity/generation/content-hash plan +and its exact completion, separately from payload completion at sequence 1. +Owner/plan mismatch, duplicate roles or inode identities, supplied paths, incomplete +payloads and inconsistent bytes refuse. The App's additive maintenance port commits +the immutable plan before its guarded callback, serializes it against reference +writers, re-attests receipt/native fences/references, and returns zero without +calling an observed completed callback again. Legacy trait implementations default +to unavailable without invoking the callback. A migration marker makes older Store +catalogs read-only; physical Store reopen and historical reprojection retain exact +phase distinctions without conferring native I/O authority. + +This is a writer prerequisite, not native retirement acceptance. No native adapter +calls the port yet; current cleanup still retains all journal/lifetime/phase +metadata. Implement native per-leaf planning/deletion/resumption and retired-receipt +lookup/reconciliation, then complete managed checkout/partial-construction and +crash/platform acceptance. M6 and all parent tasks remain open. HTTP/OpenAPI/SDK +schemas are unchanged; the Rust maintenance interface and internal event protocol +have the additive retirement state/port described above. + +The real App unavailable-policy regression initially deletes 57,344 bytes under +null configuration. The common ordinary/paired/cancellation retention guard now +refuses unavailable policy, and an actual two-payload case loses configuration after +its first unlink, preserves the second and resumes only under restored policy. +Final local evidence: 27 App reclamation/writer cases and 14 migration cases pass, +with an additional exact-inventory re-attestation and retry-counter rerun after +review refinements. The metadata callback tests deliberately prove Store sequencing +without deleting native metadata. All-target Store/server/snapshot/App/CLI Clippy, +formatting/whitespace, OpenAPI artifact and SDK generation checks pass; strict +OpenSpec passes 100 items and spec lint reports zero errors/25 warnings. macOS +sandbox deprecation and linker diagnostics were observed. Full workspace, SDK +runtime suites, native metadata deletion and new-source platform checks remain open. + +Review refinement: the metadata registry now bounds vector length and counts JSON +encoding into a 16 KiB sink before cloning/deserializing; scalar bounds also precede +typed encoding. Final oversized-owner/vector, duplicate-identity, missing/foreign +role and supplied-path cases pass, as does final server/App Clippy. This adds no +native deletion authority. Focused code/security review finds no further issue in +this writer increment; native producer/reconciliation and full branch review remain +required before delivery acceptance. + +Collection source 124c11b9 completes native run 42759174937 successfully on Linux +x64/ARM and macOS ARM/Intel. This accepts that source, not the subsequent metadata +writer protocol. Run 42759434324 at 50e00372 retains the longer workflow allowance +and is queued at this checkpoint; it also predates these writer changes. From 053dbfac78a382dc1a11212c99e63cd0247a3e98 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 02:57:59 -0400 Subject: [PATCH 64/87] Retire acknowledged snapshot metadata through exact native plans --- crates/cyber-app/src/snapshot_reclamation.rs | 7 + .../src/snapshot_reclamation/metadata.rs | 29 ++ .../cyber-app/tests/snapshot_reclamation.rs | 14 +- .../support/snapshot_native_retirement.rs | 458 ++++++++++++++++++ .../tests/support/snapshot_retirement.rs | 55 +++ .../src/runtime/snapshot_preparation.rs | 5 + .../runtime/snapshot_reclamation/metadata.rs | 41 ++ .../src/runtime/snapshot_recovery.rs | 6 + .../src/runtime/snapshot_restore_receipt.rs | 5 + crates/cyber-snapshot/src/copy/reclamation.rs | 1 + .../src/copy/reclamation/retirement.rs | 118 +++++ .../src/copy/reclamation/retirement/native.rs | 412 ++++++++++++++++ crates/cyber-snapshot/src/lib.rs | 115 ++++- crates/cyber-snapshot/src/reclamation.rs | 15 + docs/goal-delivery.md | 23 +- .../managed-recovery.md | 26 + .../complete-daily-coding-surfaces/tasks.md | 54 +++ 17 files changed, 1374 insertions(+), 10 deletions(-) create mode 100644 crates/cyber-app/tests/support/snapshot_native_retirement.rs create mode 100644 crates/cyber-snapshot/src/copy/reclamation/retirement.rs create mode 100644 crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs diff --git a/crates/cyber-app/src/snapshot_reclamation.rs b/crates/cyber-app/src/snapshot_reclamation.rs index 10a96afb7..24afbe8f6 100644 --- a/crates/cyber-app/src/snapshot_reclamation.rs +++ b/crates/cyber-app/src/snapshot_reclamation.rs @@ -124,6 +124,13 @@ impl Policy for Reclamation { .transaction(move |db| collect(&records::references(db)?).map_err(invalid)) .map_err(|error| error.to_string()) } + fn retirement_batch( + &self, + directory: Option<&str>, + after: Option<&str>, + ) -> Result { + metadata::batch(self, directory, after) + } fn refused(&self, directory: &str, error: &str) { cyber_core::log::warn( "snapshot", diff --git a/crates/cyber-app/src/snapshot_reclamation/metadata.rs b/crates/cyber-app/src/snapshot_reclamation/metadata.rs index 1a0c123a3..36693da97 100644 --- a/crates/cyber-app/src/snapshot_reclamation/metadata.rs +++ b/crates/cyber-app/src/snapshot_reclamation/metadata.rs @@ -2,6 +2,35 @@ use super::*; use records::metadata::{self, Plan as MetadataPlan}; +pub(super) fn batch( + owner: &Reclamation, + directory: Option<&str>, + after: Option<&str>, +) -> Result { + let directory = directory.map(str::to_owned); + let after = after.map(str::to_owned); + owner.0.read(move |db| { + let tx = db.unchecked_transaction()?; + let mut query = tx.prepare("SELECT e.aggregate_id,json_extract(e.data,'$.receipt') FROM event e WHERE e.type=?1 AND e.seq=0 AND e.aggregate_id>coalesce(?2,'') AND EXISTS(SELECT 1 FROM event c WHERE c.aggregate_id=e.aggregate_id AND c.seq=1 AND c.type=?3) AND NOT EXISTS(SELECT 1 FROM event c WHERE c.aggregate_id=e.aggregate_id AND c.seq=3 AND c.type=?4) AND (?5 IS NULL OR coalesce(json_extract(e.data,'$.receipt.origin.binding.directory'),json_extract(e.data,'$.receipt.checkout.resolution_origin.binding.directory'))=?5) ORDER BY e.aggregate_id LIMIT 32")?; + let mut rows = query.query(rusqlite::params![records::PLANNED, after, records::COMPLETED, metadata::COMPLETED, directory])?; + let mut receipts = Vec::new(); + let mut last = None; + while let Some(row) = rows.next()? { + let id: String = row.get(0)?; + let raw = row.get_ref(1)?.as_str().map_err(|_| invalid("Invalid retirement receipt encoding"))?; + if raw.len() > 64 * 1024 { return Err(invalid("Metadata retirement receipt exceeds discovery bound")); } + let receipt: records::Receipt = serde_json::from_str(raw).map_err(invalid)?; + if Plan::id(&receipt)? != id { return Err(invalid("Foreign retirement discovery owner")); } + receipt.owners()?; + last = Some(id); + receipts.push(receipt); + } + let after = if receipts.len() == 32 { last } else { None }; + drop(rows); drop(query); tx.commit()?; + Ok(cyber_snapshot::reclamation::RetirementBatch { receipts, after }) + }).map_err(|error| error.to_string()) +} + pub(super) fn retire( owner: &Reclamation, payload: Plan, diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs index a7326cb73..9a80951fc 100644 --- a/crates/cyber-app/tests/snapshot_reclamation.rs +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -1,5 +1,7 @@ #![cfg(any(target_os = "macos", target_os = "linux"))] //! Real App restores/receipts and native private payloads; no source writes by cleanup. +#[path = "support/snapshot_native_retirement.rs"] +mod native_retirement; #[path = "support/snapshot_retirement.rs"] mod retirement; #[path = "support/scripted_model.rs"] @@ -318,10 +320,16 @@ async fn periodic_collection_applies_the_same_retention_and_receipt_admission() f.age(); f.snapshots.gc().await; assert!(f.record().unwrap().completed); - assert!(!f.journal.join("stage_0").exists() && !f.journal.join("stage_1").exists()); + assert!(f.record().unwrap().retired && !f.journal.exists()); + let metadata = f.record().unwrap().metadata.unwrap(); assert_eq!( - std::fs::read(f.journal.join("intent.json")).unwrap(), - intent + metadata + .files + .iter() + .find(|file| file.role == records::metadata::Role::Intent) + .unwrap() + .hash, + blake3::hash(&intent).to_hex().to_string() ); assert_eq!(image_bytes(&files), images); f.app.runtime.shutdown().await; diff --git a/crates/cyber-app/tests/support/snapshot_native_retirement.rs b/crates/cyber-app/tests/support/snapshot_native_retirement.rs new file mode 100644 index 000000000..6c99cb1f5 --- /dev/null +++ b/crates/cyber-app/tests/support/snapshot_native_retirement.rs @@ -0,0 +1,458 @@ +//! Actual native metadata retirement; retained receipts never replay source work. +use super::*; + +async fn payload_complete(f: &Fixture) { + f.age(); + assert!(f.reclaim().await.unwrap() > 0); + assert!(f.record().unwrap().completed); +} +async fn retire(f: &Fixture) -> Result { + f.snapshots + .retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await +} +fn worker_routes(f: &Fixture) -> [PathBuf; 2] { + let lifetime = &f.receipt.checkout.as_ref().unwrap().lifetime; + let directory = f + .journal + .parent() + .unwrap() + .parent() + .unwrap() + .join("workers"); + [ + directory.join(&lifetime.owner.activity_id), + directory.join(lifetime.source_phase.as_ref().unwrap()), + ] +} + +#[tokio::test] +async fn native_retirement_removes_exact_metadata_and_acknowledges_from_the_retained_store_receipt() +{ + use std::os::unix::fs::PermissionsExt; + let f = Fixture::new().await; + payload_complete(&f).await; + let routes = worker_routes(&f); + let inference = f.model.requests().len(); + let messages = f + .client + .get(&format!("/sessions/{}/messages?order=asc", f.session)) + .await + .unwrap(); + let owner = f.project.parent().unwrap().join("later-owner"); + std::fs::write(&owner, b"later owner source bytes").unwrap(); + std::fs::remove_file(f.project.join("file_0")).unwrap(); + std::os::unix::fs::symlink(&owner, f.project.join("file_0")).unwrap(); + std::fs::set_permissions(&owner, std::fs::Permissions::from_mode(0o0)).unwrap(); + assert!(retire(&f).await.unwrap() > 0); + assert!(f.record().unwrap().retired); + assert!(!f.journal.exists()); + for path in routes { + assert!(!path.exists(), "{}", path.display()); + } + assert_eq!(retire(&f).await.unwrap(), 0); + let receipt = f + .client + .post( + &format!( + "/sessions/{}/rewind/restores/{}/acknowledge", + f.session, f.receipt.origin.activity_id + ), + json!({}), + ) + .await + .unwrap(); + assert_eq!( + serde_json::from_value::(receipt["data"].clone()).unwrap(), + f.receipt + ); + assert_eq!(std::fs::read_link(f.project.join("file_0")).unwrap(), owner); + std::fs::set_permissions(&owner, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!(std::fs::read(owner).unwrap(), b"later owner source bytes"); + assert_eq!( + f.client + .get(&format!("/sessions/{}/messages?order=asc", f.session)) + .await + .unwrap(), + messages + ); + assert_eq!(f.model.requests().len(), inference); + f.app.runtime.shutdown().await; +} + +struct Cut { + inner: Arc, + after: bool, + tamper: Option, +} +impl Policy for Cut { + fn receipt( + &self, + origin: &cyber_server::runtime::SnapshotOrigin, + ) -> Result, String> { + self.inner.receipt(origin) + } + fn record(&self, receipt: &records::Receipt) -> Result, String> { + self.inner.record(receipt) + } + fn reclaim(&self, plan: records::Plan, delete: Delete) -> Result { + self.inner.reclaim(plan, delete) + } + fn retire( + &self, + payload: records::Plan, + plan: records::metadata::Plan, + delete: Delete, + ) -> Result { + let after = self.after; + let tamper = self.tamper.clone(); + self.inner.retire( + payload, + plan, + Box::new(move || { + if let Some(path) = tamper { + std::fs::write(path, b"later owner metadata").unwrap(); + return delete(); + } + if after { + delete()?; + } + Err("injected metadata interruption".into()) + }), + ) + } + fn collect(&self, collect: Collect) -> Result<(), String> { + self.inner.collect(collect) + } + fn refused(&self, directory: &str, error: &str) { + self.inner.refused(directory, error); + } +} + +#[tokio::test] +async fn native_retirement_resumes_before_and_after_deletion_without_rewriting_acknowledgement() { + for after in [false, true] { + let f = Fixture::new().await; + payload_complete(&f).await; + let cut = maintenance( + &f.paths, + f.config.clone(), + Arc::new(Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after, + tamper: None, + }), + ); + assert!( + cut.retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await + .is_err() + ); + assert_eq!(f.journal.exists(), !after); + let pending = f.record().unwrap(); + assert!(pending.metadata.is_some() && !pending.retired); + let blocked = f + .app + .runtime + .snapshot_restore_acknowledge(&f.session, &f.receipt.origin.activity_id) + .await + .unwrap_err(); + assert!( + blocked.to_string().contains("retirement is pending"), + "{blocked}" + ); + assert!(retire(&f).await.unwrap() > 0); + assert!(f.record().unwrap().retired && !f.journal.exists()); + assert_eq!(retire(&f).await.unwrap(), 0); + f.app.runtime.shutdown().await; + } +} + +#[tokio::test] +async fn native_retirement_preserves_changed_or_unknown_metadata_after_planning() { + for leaf in ["intent.next", "owner-kept"] { + let f = Fixture::new().await; + payload_complete(&f).await; + let active = std::fs::read(f.journal.join("intent.json")).unwrap(); + let target = f.journal.join(leaf); + let cut = maintenance( + &f.paths, + f.config.clone(), + Arc::new(Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after: false, + tamper: Some(target.clone()), + }), + ); + assert!( + cut.retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await + .is_err() + ); + assert!(f.record().unwrap().metadata.is_some() && !f.record().unwrap().retired); + assert_eq!( + std::fs::read(f.journal.join("intent.json")).unwrap(), + active + ); + assert_eq!(std::fs::read(target).unwrap(), b"later owner metadata"); + assert!(worker_routes(&f).iter().all(|path| path.exists())); + f.app.runtime.shutdown().await; + } +} + +#[tokio::test] +async fn native_retirement_policy_loss_after_one_unlink_preserves_the_remaining_metadata() { + let f = Fixture::new().await; + payload_complete(&f).await; + let active = f.journal.join("intent.json"); + let cut = GitSnapshots::with_reclamation( + f.paths.data.clone(), + Arc::new(move |_| { + if active.exists() { + json!({}) + } else { + Value::Null + } + }), + vec![ + f.paths.config.clone(), + f.paths.state.clone(), + f.paths.cache.clone(), + f.paths.tmp.clone(), + ], + None, + Some(cyber_app::snapshot_reclamation::policy(f.app.store.clone())), + ); + assert!( + cut.retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await + .unwrap_err() + .contains("retention") + ); + assert!(!f.journal.join("intent.json").exists() && f.journal.join("intent.next").exists()); + assert!(worker_routes(&f).iter().all(|path| path.exists())); + assert!(!f.record().unwrap().retired); + assert!(retire(&f).await.unwrap() > 0); + assert!(f.record().unwrap().retired && !f.journal.exists()); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn native_metadata_retirement_frees_cap_space_without_source_or_inference_replay() { + let f = Fixture::new().await; + payload_complete(&f).await; + let namespace = f.journal.parent().unwrap().parent().unwrap(); + let cap = private_bytes(namespace) + 4096; + *f.config.lock().unwrap() = json!({"snapshots":{"retention_days":7,"fallback_max_bytes":cap}}); + let inference = f.model.requests().len(); + std::fs::write(f.project.join("zz_new"), vec![83u8; 8192]).unwrap(); + let captured = f + .snapshots + .track(f.project.to_str().unwrap()) + .await + .unwrap(); + assert!(captured.is_some()); + assert!(f.record().unwrap().retired && !f.journal.exists()); + assert_eq!( + std::fs::read(f.project.join("file_0")).unwrap(), + "before\n".repeat(4096).as_bytes() + ); + assert_eq!(f.model.requests().len(), inference); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn native_retirement_refuses_unknown_replaced_aliased_and_live_metadata_before_planning() { + use std::os::unix::fs::PermissionsExt; + for mode in ["unknown", "replacement", "symlink", "hardlink", "live"] { + let f = Fixture::new().await; + payload_complete(&f).await; + let spare = f.journal.join("intent.next"); + let retained = f.paths.tmp.join("owner-metadata"); + let mut held = None; + match mode { + "unknown" => std::fs::write(f.journal.join("owner-kept"), b"owner bytes").unwrap(), + "replacement" => { + let bytes = std::fs::read(&spare).unwrap(); + std::fs::rename(&spare, &retained).unwrap(); + std::fs::write(&spare, bytes).unwrap(); + std::fs::set_permissions(&spare, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + "symlink" => { + std::fs::rename(&spare, &retained).unwrap(); + std::os::unix::fs::symlink(&retained, &spare).unwrap(); + } + "hardlink" => std::fs::hard_link(&spare, &retained).unwrap(), + "live" => { + let file = std::fs::File::open(&worker_routes(&f)[0]).unwrap(); + file.try_lock().unwrap(); + held = Some(file); + } + _ => unreachable!(), + } + let intent = std::fs::read(f.journal.join("intent.json")).unwrap(); + assert!(retire(&f).await.is_err(), "{mode}"); + assert!(f.record().unwrap().metadata.is_none(), "{mode}"); + assert_eq!( + std::fs::read(f.journal.join("intent.json")).unwrap(), + intent + ); + assert!(worker_routes(&f).iter().all(|path| path.exists())); + drop(held); + f.app.runtime.shutdown().await; + } +} + +#[tokio::test] +async fn native_retirement_respects_retention_and_reader_leases_before_planning() { + let f = Fixture::new().await; + payload_complete(&f).await; + for settings in [ + Value::Null, + json!({"snapshots":false}), + json!({"snapshots":{"retention_days":0}}), + ] { + *f.config.lock().unwrap() = settings; + assert_eq!(retire(&f).await.unwrap(), 0); + assert!(f.record().unwrap().metadata.is_none() && f.journal.exists()); + } + *f.config.lock().unwrap() = json!({}); + let reader = cyber_server::workspace::mutation_lock(&f.project) + .unwrap() + .lock_owned() + .await; + assert!(retire(&f).await.unwrap_err().contains("reader or writer")); + assert!(f.record().unwrap().metadata.is_none()); + drop(reader); + assert!(retire(&f).await.unwrap() > 0); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn native_metadata_retirement_child() { + let Ok(control) = std::env::var("CYBER_NATIVE_METADATA_RETIREMENT_CHILD") else { + return; + }; + let control = PathBuf::from(control); + let f = Fixture::in_directory(tempfile::tempdir_in(&control).unwrap()).await; + payload_complete(&f).await; + std::fs::write(control.join("fixture.json"), json!({"root":f.paths.data.parent().unwrap(),"project":f.project,"journal":f.journal,"receipt":f.receipt}).to_string()).unwrap(); + let active = f.journal.join("intent.json"); + let marker = control.join("ready"); + let snapshots = GitSnapshots::with_reclamation( + f.paths.data.clone(), + Arc::new(move |_| { + if !active.exists() { + std::fs::write( + &marker, + b"active intent removed; remaining metadata retained", + ) + .unwrap(); + loop { + std::thread::sleep(Duration::from_secs(1)); + } + } + json!({}) + }), + vec![ + f.paths.config.clone(), + f.paths.state.clone(), + f.paths.cache.clone(), + f.paths.tmp.clone(), + ], + None, + Some(cyber_app::snapshot_reclamation::policy(f.app.store.clone())), + ); + snapshots + .retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await + .unwrap(); + panic!("worker passed the metadata cut"); +} + +#[tokio::test] +async fn native_retirement_after_real_process_death_reopens_the_exact_plan_without_source_replay() { + use std::os::unix::process::ExitStatusExt; + let mut control = tempfile::tempdir().unwrap(); + let log = std::fs::File::create(control.path().join("worker.log")).unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "native_retirement::native_metadata_retirement_child", + "--nocapture", + ]) + .env("CYBER_NATIVE_METADATA_RETIREMENT_CHILD", control.path()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let ready = loop { + if control.path().join("ready").exists() { + break true; + } + if child.try_wait().unwrap().is_some() || std::time::Instant::now() >= deadline { + break false; + } + tokio::time::sleep(Duration::from_millis(20)).await; + }; + let _ = child.kill(); + let status = child.wait().unwrap(); + if !ready { + control.disable_cleanup(true); + } + assert!( + ready, + "worker missed native metadata cut; fixture retained at {}: {}", + control.path().display(), + std::fs::read_to_string(control.path().join("worker.log")).unwrap() + ); + assert_eq!(status.signal(), Some(9)); + resume_child_fixture(control.path()).await; +} +async fn resume_child_fixture(control: &std::path::Path) { + let fixture: Value = + serde_json::from_slice(&std::fs::read(control.join("fixture.json")).unwrap()).unwrap(); + let root = PathBuf::from(fixture["root"].as_str().unwrap()); + let project = PathBuf::from(fixture["project"].as_str().unwrap()); + let journal = PathBuf::from(fixture["journal"].as_str().unwrap()); + let receipt: SnapshotRestoreReceipt = + serde_json::from_value(fixture["receipt"].clone()).unwrap(); + assert!(!journal.join("intent.json").exists() && journal.join("intent.next").exists()); + let target = root.join("later-owner"); + std::fs::write(&target, b"owner after process death").unwrap(); + std::fs::remove_file(project.join("file_0")).unwrap(); + std::os::unix::fs::symlink(&target, project.join("file_0")).unwrap(); + let paths = Paths { + data: root.join("data"), + config: root.join("config"), + state: root.join("state"), + cache: root.join("cache"), + tmp: root.join("tmp"), + }; + let store = Arc::new( + cyber_store::Store::open(cyber_store::StoreOptions::new( + DatabaseLocation::File(paths.data.join("cyber.db")), + cyber_server::runtime::Runtime::registry(), + )) + .unwrap(), + ); + let policy = cyber_app::snapshot_reclamation::policy(store); + let retained: records::Receipt = receipt.into(); + let before = policy.record(&retained).unwrap().unwrap(); + assert!(before.metadata.is_some() && !before.retired); + let snapshots = maintenance(&paths, Arc::new(Mutex::new(json!({}))), policy.clone()); + // The missing active intent cannot be discovered from journal JSON. The + // production GC resumes it from the Store's bounded retirement page. + snapshots.gc().await; + assert!(policy.record(&retained).unwrap().unwrap().retired && !journal.exists()); + assert_eq!( + snapshots + .retire(project.to_str().unwrap(), retained) + .await + .unwrap(), + 0 + ); + assert_eq!(std::fs::read_link(project.join("file_0")).unwrap(), target); + assert_eq!(std::fs::read(target).unwrap(), b"owner after process death"); +} diff --git a/crates/cyber-app/tests/support/snapshot_retirement.rs b/crates/cyber-app/tests/support/snapshot_retirement.rs index 22d87d3db..2c25b160c 100644 --- a/crates/cyber-app/tests/support/snapshot_retirement.rs +++ b/crates/cyber-app/tests/support/snapshot_retirement.rs @@ -428,3 +428,58 @@ async fn legacy_maintenance_policy_refuses_metadata_work_without_invoking_the_ca assert!(f.journal.join("intent.json").exists()); f.app.runtime.shutdown().await; } + +#[tokio::test] +async fn retirement_discovery_pages_stably_and_filters_location_without_granting_native_authority() +{ + let f = Fixture::new().await; + payload_complete(&f).await; + let prototype = f.record().unwrap().plan; + // Pagination exercises only stored index discovery. These deliberately + // uncommitted receipt variants cannot authorize any native retirement. + let mut expected = + std::collections::BTreeSet::from([records::Plan::id(&prototype.receipt).unwrap()]); + for index in 1..=32u32 { + let mut payload = prototype.clone(); + let records::Receipt::Restore(receipt) = &mut payload.receipt else { + unreachable!() + }; + receipt.events_digest = format!("{index:064x}"); + let id = records::Plan::id(&payload.receipt).unwrap(); + expected.insert(id.clone()); + let bytes = payload.total_bytes().unwrap(); + f.app.store.transaction(move |db| { + db.execute("INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES (?1,?2,0,?3,?4,1)", + rusqlite::params![format!("evt_meta_plan_{index}"),id,records::PLANNED,json!(payload).to_string()])?; + db.execute("INSERT INTO event(id,aggregate_id,seq,type,data,time) VALUES (?1,?2,1,?3,?4,1)", + rusqlite::params![format!("evt_meta_done_{index}"),id,records::COMPLETED,json!(records::Completion {plan_id:id.clone(),removed_bytes:bytes}).to_string()])?; + Ok(()) + }).unwrap(); + } + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + let first = policy + .retirement_batch(Some(f.project.to_str().unwrap()), None) + .unwrap(); + assert_eq!(first.receipts.len(), 32); + let next = policy + .retirement_batch(Some(f.project.to_str().unwrap()), first.after.as_deref()) + .unwrap(); + assert_eq!(next.receipts.len(), 1); + assert!(next.after.is_none()); + let actual = first + .receipts + .into_iter() + .chain(next.receipts) + .map(|receipt| records::Plan::id(&receipt).unwrap()) + .collect::>(); + assert_eq!(actual, expected); + assert!( + policy + .retirement_batch(Some("/another-location"), None) + .unwrap() + .receipts + .is_empty() + ); + assert!(f.journal.join("intent.json").exists()); + f.app.runtime.shutdown().await; +} diff --git a/crates/cyber-server/src/runtime/snapshot_preparation.rs b/crates/cyber-server/src/runtime/snapshot_preparation.rs index ef6a23e3d..a9beaa1c7 100644 --- a/crates/cyber-server/src/runtime/snapshot_preparation.rs +++ b/crates/cyber-server/src/runtime/snapshot_preparation.rs @@ -791,6 +791,11 @@ impl Runtime { if receipt.origin.is_none() && receipt.checkout_resources_digest.is_none() { return Ok(receipt); } + if self.snapshot_metadata_retired(&super::snapshot_reclamation::Receipt::Preparation( + Box::new(receipt.clone()), + ))? { + return Ok(receipt); + } let worker = self .preparation_worker(session, activity, Some(receipt.clone())) .await?; diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs b/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs index 53db951b1..9aa14dcde 100644 --- a/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs +++ b/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs @@ -330,3 +330,44 @@ pub(super) fn project(db: &Connection, event: &StoredEvent) -> Result<(), String } Ok(()) } + +impl crate::runtime::Runtime { + /// An observed retirement answers metadata acknowledgement without reopening + /// removed files. A pending plan freezes them against competing rewrites. + pub(crate) fn snapshot_metadata_retired( + &self, + receipt: &Receipt, + ) -> Result { + let retained = receipt.clone(); + let record = self.inner.store.read(move |db| { + let tx = db.unchecked_transaction()?; + retained.verify(&tx)?; + let record = super::lookup(&tx, &retained)?; + tx.commit()?; + Ok(record) + })?; + let Some(record) = record else { + return Ok(false); + }; + if record.retired { + let session = &receipt.origin()?.session_id; + let handle = self + .inner + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(session) + .cloned(); + if let Some(handle) = handle { + for owner in receipt.owners()? { + handle.acknowledge_snapshot_uncertainty(&owner.origin.activity_id); + } + } + return Ok(true); + } + if record.metadata.is_some() { + return Err(crate::runtime::RuntimeError::Conflict("Snapshot metadata retirement is pending; resume maintenance before acknowledgement".into())); + } + Ok(false) + } +} diff --git a/crates/cyber-server/src/runtime/snapshot_recovery.rs b/crates/cyber-server/src/runtime/snapshot_recovery.rs index 70e9a11ac..0d963149e 100644 --- a/crates/cyber-server/src/runtime/snapshot_recovery.rs +++ b/crates/cyber-server/src/runtime/snapshot_recovery.rs @@ -24,6 +24,12 @@ impl Runtime { .receipt .clone() .ok_or_else(|| RuntimeError::Corrupt("Committed recovery lacks its receipt".into()))?; + if self.snapshot_metadata_retired(&super::snapshot_reclamation::Receipt::Recovery( + receipt.clone(), + ))? { + result.journal_acknowledged = Some(true); + return Ok(result); + } let origin = result.resolution.clone().ok_or_else(|| { RuntimeError::Corrupt("Committed recovery lacks its native origin".into()) })?; diff --git a/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs index 5aae3431b..46f1ff2a8 100644 --- a/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs +++ b/crates/cyber-server/src/runtime/snapshot_restore_receipt.rs @@ -100,6 +100,11 @@ impl Runtime { .ok_or_else(|| { RuntimeError::Conflict("No immutable committed restore receipt".into()) })?; + if self.snapshot_metadata_retired(&super::snapshot_reclamation::Receipt::Restore( + receipt.clone(), + ))? { + return Ok(receipt); + } let handle = self.inner.handle(session).await?; let state = handle.state.lock().await; let directory = state.info.directory.clone(); diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index b0a2f1a2b..2e4b82a69 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -2,6 +2,7 @@ mod cancelled; mod checkouts; mod paired; +pub(crate) mod retirement; use super::{Store, ownership, restore::Journal, storage}; use crate::reclamation::{Policy, Settlement}; use cap_fs_ext::DirExt; diff --git a/crates/cyber-snapshot/src/copy/reclamation/retirement.rs b/crates/cyber-snapshot/src/copy/reclamation/retirement.rs new file mode 100644 index 000000000..0a8e87d62 --- /dev/null +++ b/crates/cyber-snapshot/src/copy/reclamation/retirement.rs @@ -0,0 +1,118 @@ +//! Exact private metadata retirement after durable payload completion. +mod native; +use super::*; +use cyber_server::runtime::snapshot_reclamation::{self as records, Receipt, metadata}; + +pub(crate) fn retire( + store: Store, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + policy: Arc, + receipt: Receipt, +) -> Result { + let record = policy + .record(&receipt)? + .ok_or("Missing completed payload cleanup")?; + if !record.completed { + return Err("Metadata retirement needs observed payload completion".into()); + } + if record.retired { + return Ok(0); + } + let origin = receipt.origin().map_err(storage::fail)?; + let settlement = policy + .receipt(origin)? + .ok_or("Missing metadata retirement settlement")?; + if settlement.receipt != receipt || record.plan.receipt != receipt { + return Err("Metadata retirement receipt changed".into()); + } + if !due(config.as_ref(), &store.root, settlement.settled_at_ms) { + return Ok(0); + } + for owner in receipt.owners().map_err(storage::fail)? { + if owner.lifetime.checkout_resources_digest(&[])? != owner.resources_digest { + return Err("Managed checkout metadata retirement is not yet available".into()); + } + } + if record.metadata.is_none() { + verify_initial(&store, &record)?; + } + let prepared = + native::Prepared::new(store, workspace, config, settlement.settled_at_ms, record)?; + let plan = prepared.plan.clone(); + let payload = prepared.payload.clone(); + policy.retire(payload, plan, Box::new(move || prepared.delete())) +} + +fn verify_initial(store: &Store, retained: &records::Record) -> Result<(), String> { + let receipt = &retained.plan.receipt; + for owner in receipt.owners().map_err(storage::fail)? { + store.verify_worker_namespace(owner.origin)?; + let (_, journal) = store.journal(&owner.origin.binding.journal_id)?; + if journal.origin.as_ref() != Some(owner.origin) { + return Err("Metadata retirement journal owner differs".into()); + } + let (digest, admitted) = initial_journal(store, retained, &journal, owner.origin)?; + if store.worker_digest(Some(owner.origin))?.as_ref() != Some(&digest) { + return Err("Metadata retirement intent differs from durable settlement".into()); + } + store.verify_reclamation_phase_admission( + owner.lifetime, + owner.origin, + owner.resources_digest, + admitted, + )?; + } + Ok(()) +} +fn initial_journal( + store: &Store, + retained: &records::Record, + journal: &Journal, + origin: &cyber_server::runtime::SnapshotOrigin, +) -> Result<(String, bool), String> { + match &retained.plan.receipt { + Receipt::Restore(receipt) if ordinary(journal) => { + Ok((receipt.intent_digest.clone(), !journal.changes.is_empty())) + } + Receipt::Preparation(receipt) if cancelled::verify(store, receipt, journal)? => Ok(( + retained + .plan + .intent_digest + .clone() + .ok_or("Missing cancellation intent digest")?, + false, + )), + Receipt::Recovery(receipt) => { + let (_, resolution) = store.journal(&receipt.resolution_id)?; + let (_, original) = store.journal(&receipt.journal_id)?; + store.verify_settlement(&original)?; + store.verify_receipt(resolution.origin.as_ref(), receipt)?; + if !paired::eligible(&resolution) + || original.settled_by.as_deref() != Some(&receipt.resolution_id) + { + return Err("Metadata retirement recovery pair differs".into()); + } + if origin == &receipt.original { + Ok(( + resolution + .resolution + .as_ref() + .ok_or("Missing recovery intent")? + .original_digest + .clone(), + true, + )) + } else { + Ok(( + receipt + .resolution_digest + .clone() + .ok_or("Missing resolution digest")?, + !resolution.changes.is_empty(), + )) + } + } + _ => Err("Metadata retirement journal is not acknowledged".into()), + } +} diff --git a/crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs b/crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs new file mode 100644 index 000000000..ecb294ccf --- /dev/null +++ b/crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs @@ -0,0 +1,412 @@ +//! Native guards and fixed semantic routes; no source image is opened or installed. +use super::*; +use cyber_server::runtime::{SnapshotLifetimeBinding, SnapshotOrigin}; +use metadata::{File as Entry, Plan as MetadataPlan, Role}; +use std::os::unix::fs::FileExt; + +struct Owner { + origin: SnapshotOrigin, + binding: SnapshotLifetimeBinding, + journal: Option, + journal_generation: Option, + _lifetime: Option, +} +struct Leaf { + entry: Entry, + owner: usize, + name: String, + pin: Option, +} +pub(super) struct Prepared { + pub plan: MetadataPlan, + pub payload: records::Plan, + store: Store, + workers: Dir, + workers_generation: String, + owners: Vec, + leaves: Vec, + _workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + settled: i64, +} + +impl Prepared { + pub fn new( + store: Store, + workspace: tokio::sync::OwnedMutexGuard<()>, + config: Arc, + settled: i64, + record: records::Record, + ) -> Result { + let frozen = record.metadata; + if let Some(plan) = &frozen { + plan.validate(&record.plan).map_err(storage::fail)?; + } + let workers = storage::existing_child(&store.namespace, "workers", true)?; + let mut owners = Vec::new(); + for evidence in record.plan.receipt.owners().map_err(storage::fail)? { + validate_names(evidence.origin, evidence.lifetime)?; + let journal = existing_journal(&store, evidence.origin, frozen.is_some())?; + let lifetime = match workers.symlink_metadata(&evidence.lifetime.owner.activity_id) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound && frozen.is_some() => { + None + } + _ => Some(store.lock_lifetime(evidence.lifetime)?), + }; + let journal_generation = journal.as_ref().map(directory_generation).transpose()?; + owners.push(Owner { + origin: evidence.origin.clone(), + binding: evidence.lifetime.clone(), + journal, + journal_generation, + _lifetime: lifetime, + }); + } + let entries = match &frozen { + Some(plan) => plan.files.clone(), + None => capture(&store, &workers, &owners, &record.plan.receipt)?, + }; + let plan = MetadataPlan { + version: 1, + payload_plan_id: records::Plan::id(&record.plan.receipt).map_err(storage::fail)?, + files: entries, + }; + plan.validate(&record.plan).map_err(storage::fail)?; + if frozen.as_ref().is_some_and(|frozen| *frozen != plan) { + return Err("Metadata retirement plan changed".into()); + } + let mut leaves = Vec::new(); + for entry in &plan.files { + let owner = owners + .iter() + .position(|owner| owner.origin.activity_id == entry.owner) + .ok_or("Foreign metadata worker")?; + let name = name(entry.role, &owners[owner])?.to_owned(); + let dir = leaf_directory(&owners[owner], &workers, entry.role); + let pin = pin_entry(dir, &name, entry, frozen.is_some())?; + leaves.push(Leaf { + entry: entry.clone(), + owner, + name, + pin, + }); + } + leaves.sort_by_key(|leaf| priority(leaf.entry.role)); + let prepared = Self { + workers_generation: directory_generation(&workers)?, + store, + workers, + owners, + leaves, + plan, + payload: record.plan, + _workspace: workspace, + config, + settled, + }; + prepared.verify()?; + Ok(prepared) + } + fn verify(&self) -> Result<(), String> { + self.store.verify_root()?; + if !due(self.config.as_ref(), &self.store.root, self.settled) { + return Err("Snapshot retention no longer permits metadata retirement".into()); + } + if !cyber_core::worktrees::Repository::managed_locations_at(&self.store.root) + .map_err(storage::fail)? + .is_empty() + { + return Err("Metadata retirement checkout registration changed".into()); + } + let fresh = Store::existing( + &self.store.data_path, + &self.store.root.to_string_lossy(), + self.store.options.clone(), + )? + .ok_or("Metadata namespace disappeared")?; + let fresh_workers = storage::existing_child(&fresh.namespace, "workers", true)?; + for owner in &self.owners { + self.verify_owner(owner, &fresh, &fresh_workers)?; + } + if directory_generation(&self.workers)? != self.workers_generation { + return Err("Metadata worker directory changed".into()); + } + for leaf in &self.leaves { + let dir = leaf_directory(&self.owners[leaf.owner], &self.workers, leaf.entry.role); + match (&leaf.pin, dir) { + (Some(pin), Some(dir)) => pin.verify(dir, &leaf.name)?, + (None, Some(dir)) => absent(dir, &leaf.name)?, + (None, None) => {} + _ => return Err("Metadata journal disappeared during retirement".into()), + } + } + self.inventory() + } + fn verify_owner( + &self, + owner: &Owner, + fresh: &Store, + fresh_workers: &Dir, + ) -> Result<(), String> { + self.store.verify_lock_route( + owner + .origin + .binding + .lock_identity + .as_deref() + .ok_or("Legacy metadata lock binding")?, + )?; + if owner.origin.binding.directory != self.store.root + || owner.origin.binding.root_identity != self.store.root_id + || owner.binding.root_identity != self.store.root_id + || owner.binding.namespace_identity != storage::dir_identity(&fresh.namespace)? + || owner.binding.namespace_identity != storage::dir_identity(&self.store.namespace)? + || owner.binding.directory_identity != storage::dir_identity(fresh_workers)? + || owner.binding.directory_identity != storage::dir_identity(&self.workers)? + || owner.origin.binding.namespace_identity != storage::dir_identity(&fresh.journals)? + || owner.origin.binding.namespace_identity + != storage::dir_identity(&self.store.journals)? + { + return Err("Metadata retirement namespace route changed".into()); + } + match &owner.journal { + Some(journal) => { + let route = storage::existing_child( + &fresh.journals, + &owner.origin.binding.journal_id, + true, + )?; + if storage::dir_identity(&route)? != owner.origin.binding.journal_identity + || storage::dir_identity(journal)? != owner.origin.binding.journal_identity + || Some(directory_generation(journal)?) != owner.journal_generation + { + return Err("Metadata retirement journal route or generation changed".into()); + } + } + None => absent(&fresh.journals, &owner.origin.binding.journal_id)?, + } + Ok(()) + } + fn inventory(&self) -> Result<(), String> { + for (index, owner) in self.owners.iter().enumerate() { + if let Some(journal) = &owner.journal { + let known = self + .leaves + .iter() + .filter(|leaf| leaf.owner == index && journal_role(leaf.entry.role)) + .map(|leaf| leaf.name.as_str()) + .collect::>(); + if storage::names(journal, super::super::super::ENTRY_LIMIT * 3)? + .iter() + .any(|name| !known.contains(name.as_str())) + { + return Err("Unknown journal entry prevents metadata retirement".into()); + } + } + } + Ok(()) + } + pub fn delete(mut self) -> Result { + self.verify()?; + for index in 0..self.leaves.len() { + self.verify()?; + let leaf = &self.leaves[index]; + if leaf.pin.is_some() { + let dir = leaf_directory(&self.owners[leaf.owner], &self.workers, leaf.entry.role) + .ok_or("Missing metadata directory")?; + dir.remove_file(&leaf.name).map_err(storage::fail)?; + storage::sync(dir)?; + let generation = directory_generation(dir)?; + if journal_role(leaf.entry.role) { + self.owners[leaf.owner].journal_generation = Some(generation); + } else { + self.workers_generation = generation; + } + self.leaves[index].pin = None; + } + } + for index in 0..self.owners.len() { + self.verify()?; + if self.owners[index].journal.is_some() { + self.store + .journals + .remove_dir(&self.owners[index].origin.binding.journal_id) + .map_err(storage::fail)?; + storage::sync(&self.store.journals)?; + self.owners[index].journal = None; + self.owners[index].journal_generation = None; + } + } + self.verify()?; + self.plan.total_bytes().map_err(storage::fail) + } +} + +fn journal_role(role: Role) -> bool { + matches!(role, Role::Intent | Role::Spare | Role::Cancellation) +} +fn priority(role: Role) -> u8 { + match role { + Role::Intent => 0, + Role::Spare => 1, + Role::Cancellation => 2, + Role::SourcePhase => 3, + Role::Lifetime => 4, + } +} +fn name(role: Role, owner: &Owner) -> Result<&str, String> { + match role { + Role::Intent => Ok("intent.json"), + Role::Spare => Ok("intent.next"), + Role::Cancellation => Ok("preparation.json"), + Role::Lifetime => Ok(&owner.binding.owner.activity_id), + Role::SourcePhase => owner + .binding + .source_phase + .as_deref() + .ok_or("Missing metadata source phase".into()), + } +} +fn leaf_directory<'a>(owner: &'a Owner, workers: &'a Dir, role: Role) -> Option<&'a Dir> { + if journal_role(role) { + owner.journal.as_ref() + } else { + Some(workers) + } +} +fn directory_generation(dir: &Dir) -> Result { + ownership::generation(&dir.try_clone().map_err(storage::fail)?.into_std_file()) +} +fn absent(dir: &Dir, name: &str) -> Result<(), String> { + match dir.symlink_metadata(name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(storage::fail(error)), + _ => Err("Retired metadata route was replaced".into()), + } +} +fn validate_names( + origin: &SnapshotOrigin, + binding: &SnapshotLifetimeBinding, +) -> Result<(), String> { + let component = |name: &str, limit| { + !name.is_empty() + && name.len() <= limit + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + }; + if !origin.binding.journal_id.starts_with("restore_") + || !component(&origin.binding.journal_id, 256) + || !component(&origin.activity_id, 128) + || binding.owner.activity_id != origin.activity_id + || binding + .source_phase + .as_deref() + .and_then(|name| name.strip_prefix("phase_")) + .is_none_or(|id| id.len() != 26 || id.parse::().is_err()) + { + return Err("Invalid metadata retirement component".into()); + } + Ok(()) +} +fn existing_journal( + store: &Store, + origin: &SnapshotOrigin, + resuming: bool, +) -> Result, String> { + match store.journals.symlink_metadata(&origin.binding.journal_id) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound && resuming => Ok(None), + _ => { + let dir = storage::existing_child(&store.journals, &origin.binding.journal_id, true)?; + if storage::dir_identity(&dir)? != origin.binding.journal_identity { + return Err("Metadata journal identity changed".into()); + } + Ok(Some(dir)) + } + } +} +fn content_hash(file: &std::fs::File) -> Result { + let length = file.metadata().map_err(storage::fail)?.len(); + if length > super::super::super::RECORD_LIMIT { + return Err("Metadata file exceeds retirement bound".into()); + } + let mut hash = blake3::Hasher::new(); + let mut offset = 0; + let mut bytes = [0; 8192]; + while offset < length { + let limit = (length - offset).min(bytes.len() as u64) as usize; + let read = file + .read_at(&mut bytes[..limit], offset) + .map_err(storage::fail)?; + if read == 0 { + return Err("Metadata file shortened while hashing".into()); + } + hash.update(&bytes[..read]); + offset += read as u64; + } + Ok(hash.finalize().to_hex().to_string()) +} +fn pin_entry( + dir: Option<&Dir>, + name: &str, + entry: &Entry, + resuming: bool, +) -> Result, String> { + let Some(dir) = dir else { + return if resuming { + Ok(None) + } else { + Err("Missing metadata journal".into()) + }; + }; + if let Err(error) = dir.symlink_metadata(name) { + if error.kind() == std::io::ErrorKind::NotFound && resuming { + return Ok(None); + } + return Err(storage::fail(error)); + } + let pin = MetadataPin::open(dir, name)?; + if pin.identity != entry.identity + || pin.generation != entry.generation + || pin.file.metadata().map_err(storage::fail)?.len() != entry.bytes + || content_hash(&pin.file)? != entry.hash + { + return Err("Metadata retirement file identity, generation or contents changed".into()); + } + pin.verify(dir, name)?; + Ok(Some(pin)) +} +fn capture( + store: &Store, + workers: &Dir, + owners: &[Owner], + receipt: &Receipt, +) -> Result, String> { + let mut entries = Vec::new(); + for owner in owners { + let (_, journal) = store.journal(&owner.origin.binding.journal_id)?; + let mut roles = vec![Role::Intent, Role::SourcePhase, Role::Lifetime]; + if journal.updates.is_some() { + roles.push(Role::Spare); + } + if matches!(receipt, Receipt::Preparation(_)) { + roles.push(Role::Cancellation); + } + for role in roles { + let name = name(role, owner)?; + let dir = leaf_directory(owner, workers, role).ok_or("Missing retirement journal")?; + let pin = MetadataPin::open(dir, name)?; + let entry = Entry { + owner: owner.origin.activity_id.clone(), + role, + identity: pin.identity.clone(), + generation: pin.generation.clone(), + hash: content_hash(&pin.file)?, + bytes: pin.file.metadata().map_err(storage::fail)?.len(), + }; + pin.verify(dir, name)?; + entries.push(entry); + } + } + Ok(entries) +} diff --git a/crates/cyber-snapshot/src/lib.rs b/crates/cyber-snapshot/src/lib.rs index 895870c3d..d7f0e381a 100644 --- a/crates/cyber-snapshot/src/lib.rs +++ b/crates/cyber-snapshot/src/lib.rs @@ -47,6 +47,8 @@ pub struct GitSnapshots { references: Option>, unavailable: Mutex>, reclamation: Option>, + #[cfg(any(target_os = "macos", target_os = "linux"))] + retirement_cursor: Mutex>, } impl GitSnapshots { @@ -78,6 +80,8 @@ impl GitSnapshots { private_roots, references, unavailable: Mutex::new(HashMap::new()), + #[cfg(any(target_os = "macos", target_os = "linux"))] + retirement_cursor: Mutex::new(None), }) } @@ -367,7 +371,14 @@ impl GitSnapshots { self.file_copy(directory.to_owned(), |store| store.track()) .await } - Ok(_) => result, + Ok(_) => { + if self.retire_for_capacity(directory, policy).await > 0 { + self.file_copy(directory.to_owned(), |store| store.track()) + .await + } else { + result + } + } Err(error) => { policy.refused(directory, &error); result @@ -375,6 +386,75 @@ impl GitSnapshots { } } + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn retire_for_capacity( + &self, + directory: &str, + policy: &Arc, + ) -> u64 { + let batch = match policy.retirement_batch(Some(directory), None) { + Ok(batch) => batch, + Err(error) => { + policy.refused(directory, &error); + return 0; + } + }; + let mut freed = 0u64; + for receipt in batch.receipts { + match self.retire(directory, receipt).await { + Ok(bytes) => freed = freed.saturating_add(bytes), + Err(error) => policy.refused(directory, &error), + } + } + freed + } + + /// Retire exact private metadata for an already completed payload plan. + /// The Store receipt remains authoritative; this never replays source work. + pub async fn retire( + &self, + directory: &str, + receipt: cyber_server::runtime::snapshot_reclamation::Receipt, + ) -> Result { + let policy = self + .reclamation + .clone() + .ok_or("Snapshot retirement authority is unavailable")?; + let origin = receipt.origin().map_err(|error| error.to_string())?; + if origin.binding.directory != Path::new(directory) { + return Err("Metadata retirement belongs to another Location".into()); + } + let record = policy + .record(&receipt)? + .ok_or("Missing payload retirement receipt")?; + if record.retired { + return Ok(0); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + self.recovery_location(directory)?; + let source = directory.to_owned(); + let data = self.data_dir.clone(); + let options = self.recovery_options(directory); + let config = self.config.clone(); + let lock = cyber_server::workspace::mutation_lock(Path::new(directory)) + .map_err(|error| error.to_string())?; + let workspace = lock + .try_lock_owned() + .map_err(|_| "Snapshot reader or writer is active")?; + tokio::task::spawn_blocking(move || { + let store = copy::Store::recover_existing(&data, &source, options)?; + copy::reclamation::retirement::retire(store, workspace, config, policy, receipt) + }) + .await + .map_err(|error| error.to_string())? + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + Err("Native snapshot metadata retirement is unsupported on this host".into()) + } + } + /// Reclaim one independently settled ordinary journal. Intent/lifetime /// metadata and durable retry receipts remain retained. No source effects. pub async fn reclaim(&self, directory: &str) -> Result { @@ -458,6 +538,7 @@ impl GitSnapshots { Ok(Err(error)) => policy.refused("snapshot inventory", &error), Err(error) => policy.refused("snapshot inventory", &error.to_string()), } + self.gc_metadata(policy).await; let (data, config, policy) = (self.data_dir.clone(), self.config.clone(), policy.clone()); let report = policy.clone(); @@ -487,6 +568,38 @@ impl GitSnapshots { git::gc(&git_dir).await; } } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn gc_metadata(&self, policy: &Arc) { + let after = self + .retirement_cursor + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + match policy.retirement_batch(None, after.as_deref()) { + Ok(batch) => { + for receipt in batch.receipts { + let source = receipt + .origin() + .ok() + .and_then(|origin| origin.binding.directory.to_str()) + .map(str::to_owned); + if let Some(source) = source { + if let Err(error) = self.retire(&source, receipt).await { + policy.refused(&source, &error); + } + } else { + policy.refused("metadata retirement discovery", "Missing native Location"); + } + } + *self + .retirement_cursor + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = batch.after; + } + Err(error) => policy.refused("metadata retirement discovery", &error), + } + } } impl Snapshots for GitSnapshots { diff --git a/crates/cyber-snapshot/src/reclamation.rs b/crates/cyber-snapshot/src/reclamation.rs index e924db692..3001a2056 100644 --- a/crates/cyber-snapshot/src/reclamation.rs +++ b/crates/cyber-snapshot/src/reclamation.rs @@ -13,6 +13,12 @@ pub struct Settlement { pub type Delete = Box Result + Send>; pub type Collect = Box) -> Result<(), String> + Send>; +#[derive(Default)] +pub struct RetirementBatch { + pub receipts: Vec, + /// Continue after this deterministic aggregate; None completes one pass. + pub after: Option, +} /// Implementations serialize verification/deletion against reference writers. /// A durable plan must be committed before invoking `delete`; native guards are /// retained by that closure. Maintenance never acknowledges a native activity. @@ -31,5 +37,14 @@ pub trait Policy: Send + Sync { Err("Snapshot metadata retirement is unavailable".into()) } fn collect(&self, collect: Collect) -> Result<(), String>; + /// A bounded stable page also discovers partially retired journals whose + /// intent file is already gone. Legacy implementations have no candidates. + fn retirement_batch( + &self, + _directory: Option<&str>, + _after: Option<&str>, + ) -> Result { + Ok(RetirementBatch::default()) + } fn refused(&self, directory: &str, error: &str); } diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 8a4283638..97aefebb1 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -105,7 +105,7 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. restart, alias, namespace-transplant and reference regressions pass locally. This does not retire journal/worker/checkout metadata or complete M6. -8. **Metadata retirement now has a durable writer protocol, not native deletion.** +8. **Metadata retirement now joins the durable writer to exact native deletion.** Reclamation history distinguishes payload completion, an immutable metadata role/identity/generation/hash plan, and metadata completion. The App maintenance port commits the plan first, excludes reference writers during its guarded @@ -113,15 +113,20 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. Migration catalog fencing preserves older evidence; logical history survives Store reopen and reprojection after later native fences or Session deletion. The Rust maintenance port defaults to unavailable for older implementations. - No native producer invokes it yet: ordinary cleanup still retains journal, - phase and lifetime files. Native deletion/resumption, receipt reconciliation, - managed claims and partial-construction acceptance remain required by M6. + The native adapter now pins and removes acknowledged journal, phase and lifetime + files for empty checkout inventories. Periodic Store discovery resumes after + active-intent deletion, including a tested real process kill; cap pressure can + reclaim metadata before one capture retry. Completed acknowledgement returns + the retained Store receipt without reopening deleted files. Managed claims, + partial construction and complete receipt-kind/platform acceptance remain open. A separate real App regression reproduced 57,344 bytes of payload deletion when configuration resolved to null. Cleanup now preserves bytes initially and stops after policy loss between unlinks, retaining its exact resumable plan. - Twenty-seven App cases and 14 migration tests pass locally; native deletion is - not inferred from the writer callback tests. + Thirty-seven App reclamation tests pass locally, including actual native + deletion, tamper/alias/live-owner refusals, policy loss, process death and cap + retry. The earlier 14 migration tests cover the writer catalog. Fresh platform + and broader current-source verification remain required before M6 acceptance. Native run [42733865471](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42733865471) passes Linux x64/ARM and macOS ARM/Intel at `e2b3ba18`, including the earlier @@ -139,6 +144,12 @@ predates the collection-provenance continuation, whose source at `124c11b9` has its own run [42759174937](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42759174937), now successful on all four Linux/macOS platforms. It predates the metadata writer protocol and payload-policy fix; fresh source-bound acceptance remains required. +The subsequent run [42759434324](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42759434324) +at `50e00372` passes both Linux jobs and macOS ARM, but Intel fails managed Git +prepared cancellation because its preparation origin is not observed. Its private +fixture is retained/uploaded; independent client/JSONL/structured checks pass. +That acceptance failure remains unresolved. The queued writer run at `5be366b7` +and new native retirement need their own evidence. Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index f331493ee..cb6fecec4 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -980,6 +980,32 @@ after later fences or Session deletion without granting any filesystem capabilit The retained Store receipt SHALL distinguish payload completion, pending metadata retirement and observed retirement completion, including after physical Store reopen. +The native retirement entry SHALL validate acknowledged journals, immutable intent +digests, positive source phases and exited exact lifetimes before publishing a new +metadata plan. All journal entries must be known metadata; remaining payloads, +construction entries and private parent directories refuse before publication. +Each surviving planned leaf SHALL be a unique private regular file with the exact +native identity, generation, size and BLAKE3 hash. It SHALL retain the native store +lock and Location reader/mutation lease, recheck policy, registrations, routes and +pins before each unlink, synchronize affected directories and remove only the +exact empty journal directories. Missing leaves are restart progress only under +the already durable metadata plan; replacements and newly introduced entries +refuse. Lifetime files are removed last and their actual lock descriptors remain +held through the final filesystem call. + +Explicit acknowledgement SHALL return the original verified durable receipt after +observed metadata completion without reopening removed files or source images. +A pending metadata plan SHALL refuse competing acknowledgement rewrites and +require exact maintenance resumption. Neither path clears unrelated uncertainty. + +Periodic maintenance SHALL discover completed payload receipts from the Store even +after the active intent has been removed. Discovery SHALL page at most 32 receipts, +bound each encoded receipt to 64 KiB, preserve stable aggregate ordering and advance +past deferred candidates before cycling. Discovery alone grants no native authority. +A cap-triggered capture MAY attempt one bounded Location-scoped metadata batch and +retry capture once after observed reclamation; ordinary retention, reference and +native ownership checks remain mandatory. + Event validation alone SHALL NOT authorize native deletion or establish that it occurred. The native adapter must independently retain the exact namespace, reader/lifetime/checkout proof and per-leaf pins around its bounded deletion diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 94d475752..5f03e9b9e 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1060,3 +1060,57 @@ Collection source 124c11b9 completes native run 42759174937 successfully on Linu x64/ARM and macOS ARM/Intel. This accepts that source, not the subsequent metadata writer protocol. Run 42759434324 at 50e00372 retains the longer workflow allowance and is queued at this checkpoint; it also predates these writer changes. + +Native metadata-retirement continuation (2026-11-10): the Linux/macOS adapter now +captures semantic-role identities, generations, BLAKE3 hashes and exact byte totals +for acknowledged journals with empty complete checkout inventories. It retains the +Store namespace lock, Location mutation/reader lease and actual exited lifetime +locks, verifies current and retained routes, known-only journal inventory, pins and +policy before each unlink, and synchronizes exact leaf/directory removal. Existing +immutable plans resume missing leaves without reconstructing or replaying source. +Unknown entries, replacement inodes, aliases, live owners and changed policy refuse. +The Store writer remains held through the callback; completed receipts answer native +acknowledgement from retained Store history, while pending plans freeze rewrites. + +Periodic maintenance discovers at most 32 payload-completed receipts from the Store +per stable cursor page, with 64 KiB per-receipt bounds and Location-scoped cap +fallback. Thus missing active intents do not hide interrupted cleanup, and deferred +managed candidates do not permanently starve later pages. Discovery is not native +authority. A successful metadata cap fallback permits one fresh capture attempt. + +Local evidence: all 37 App reclamation cases pass, including real ordinary native +retirement, actual HTTP acknowledgement after removal, before/after callback cuts, +replacement/unknown/alias/live-lock refusal, policy loss after the first unlink, +stable discovery pagination, cap-triggered capture and actual subprocess SIGKILL +followed by physical Store reopen and production periodic resumption. Later source +symlinks/unreadable owner bytes, transcript and inference counts are preserved. +The child fixture is an inert top-level test unless explicitly invoked by its parent; +its process-death assertion actually executes in the parent case. Affected server, +snapshot and App all-target Clippy passes with warnings denied; native macOS C +sandbox deprecation diagnostics are observed without unchanged-main attribution. +Managed claim retirement, zero-payload/partial construction, old/deleted Location +maintenance, full receipt-kind cuts and current-source platform acceptance remain +required. M6 and parent tasks remain open; this is not issue or delivery completion. + +Prior-source native run 42759434324 at 50e00372 finishes both Linux jobs and macOS +ARM successfully; Intel fails the managed Git prepared-cancellation test because +its preparation origin was not observed (branches_cli.rs:1912). The failed private +fixture is retained/uploaded at .tmphzzjxr; the fixture-preservation probe's earlier +caught panic is expected and passes. Independent preparation/TUI, JSONL and +structured checks pass on Intel. This is a real unresolved acceptance failure, +not a passing matrix or evidence that a longer job allowance corrected the case. +Run 42767559419 at 5be366b7 is queued at this audit and predates native retirement. + +Final local checkpoint verification: native-git-acceptance snapshot package passes +153 library cases (8 subprocess fixtures intentionally ignored at top level), 31 +file-copy cases and 22 Git cases (1 subprocess fixture ignored at top level); the +Windows-only and doc binaries select zero here. Server snapshot selection passes +53 cases. OpenAPI artifact, SDK generation consistency, formatting and whitespace +checks pass. CI-pinned OpenSpec 1.13.2 validates all 100 items; spec lint reports zero +errors/25 warnings. Code-review-skill review of the native descriptor/route checks, +semantic-only plan, writer-held callback, refusal/resumption and actual tests finds +no additional defect in the tested ordinary path. Full receipt-kind acceptance +has a test gap: native preparation/paired retirement needs direct cases, and managed +retirement remains unsupported. These are required before accepting M6/delivery. +Full workspace, SDK runtime, current-source CLI/TUI and cross-platform acceptance +were not rerun for this checkpoint. No PR, merge or parent completion is claimed. From b4f1804c18b19b400369df5679b1fa9683658b92 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 03:07:32 -0400 Subject: [PATCH 65/87] Write repeated raw Git blobs once per capture --- .github/workflows/non-git-snapshot-proof.yml | 2 + crates/cyber-cli/tests/branches_cli.rs | 20 ++- crates/cyber-snapshot/src/git/raw.rs | 33 +++-- crates/cyber-snapshot/src/git/raw/tests.rs | 136 ++++++++++++++++++ docs/goal-delivery.md | 9 ++ .../managed-recovery.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 39 +++++ 7 files changed, 228 insertions(+), 17 deletions(-) create mode 100644 crates/cyber-snapshot/src/git/raw/tests.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index deae05724..43aad5084 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -6,7 +6,9 @@ on: - crates/cyber-app/Cargo.toml - crates/cyber-app/src/** - crates/cyber-app/tests/support/scripted_model.rs + - crates/cyber-app/tests/support/snapshot*.rs - crates/cyber-server/src/runtime/snapshot*.rs + - crates/cyber-server/src/runtime/snapshot_reclamation/** - crates/cyber-server/src/runtime/rewind*.rs - crates/cyber-server/src/runtime/drain.rs - crates/cyber-server/src/runtime/structured*.rs diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index bc68c6c83..105b3b31a 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1883,6 +1883,9 @@ fn kill_after_prepared_images(env: &Env, id: &str, first: &str) -> String { let preview = reviewed_preview(env, id, &selection); let mut body = selection; body["confirmation"] = preview["confirmation"].clone(); + // Keep request diagnostics in the failed fixture even if observation times + // out while the real service is still validating/preparing the restore. + let diagnostics = env.root.join("preparation-confirm.stderr"); let mut pending = env .command(&[ "api", @@ -1891,8 +1894,8 @@ fn kill_after_prepared_images(env: &Env, id: &str, first: &str) -> String { "-d", &body.to_string(), ]) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) + .stdout(std::fs::File::create(env.root.join("preparation-confirm.stdout")).unwrap()) + .stderr(std::fs::File::create(&diagnostics).unwrap()) .spawn() .unwrap(); let data = env.root.join("cyber-home/data"); @@ -1903,18 +1906,22 @@ fn kill_after_prepared_images(env: &Env, id: &str, first: &str) -> String { .unwrap(); let db = rusqlite::Connection::open(database).unwrap(); db.busy_timeout(Duration::from_secs(5)).unwrap(); - let deadline = std::time::Instant::now() + Duration::from_secs(45); + let started = std::time::Instant::now(); + let deadline = started + Duration::from_secs(45); let activity = loop { let activity: Option = db.query_row("SELECT aggregate_id FROM event e WHERE type='native.activity.changed.1' AND seq=1 AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_origin') IS NOT NULL AND json_extract(data,'$.snapshot_source_admitted')=0 AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq)", [id], |row|row.get(0)).optional().unwrap(); if let Some(activity) = activity { break activity; } + let status = pending.try_wait().unwrap(); assert!( - pending.try_wait().unwrap().is_none() && std::time::Instant::now() < deadline, - "preparation origin was not observed" + status.is_none() && std::time::Instant::now() < deadline, + "preparation origin was not observed; client status {status:?}: {}", + std::fs::read_to_string(&diagnostics).unwrap_or_default() ); std::thread::sleep(Duration::from_millis(1)); }; + eprintln!("preparation origin observed after {:?}", started.elapsed()); // Real SQLite writer contention holds source admission, not preparation I/O. // No row is edited and no native acknowledgement or effect hook is replaced. db.execute_batch("BEGIN IMMEDIATE").unwrap(); @@ -1955,10 +1962,9 @@ fn kill_after_prepared_images(env: &Env, id: &str, first: &str) -> String { break; } if let Some(status) = pending.try_wait().unwrap() { - let output = pending.wait_with_output().unwrap(); panic!( "restore exited before publishing prepared identities ({status}): {}", - String::from_utf8_lossy(&output.stderr) + std::fs::read_to_string(&diagnostics).unwrap_or_default() ); } assert!( diff --git a/crates/cyber-snapshot/src/git/raw.rs b/crates/cyber-snapshot/src/git/raw.rs index 7e389199d..e41a12f0e 100644 --- a/crates/cyber-snapshot/src/git/raw.rs +++ b/crates/cyber-snapshot/src/git/raw.rs @@ -23,18 +23,28 @@ impl Shadow { files: &BTreeMap, ) -> Result { let mut directories: BTreeMap> = BTreeMap::from([(String::new(), vec![])]); + // Borrow exact bytes from this already bounded, verified capture. Keep + // every path/mode entry, but write shared blob contents only once here. + let mut blobs = BTreeMap::<&[u8], String>::new(); for (path, image) in files { images::validate_path(path)?; - let object = String::from_utf8( - self.input( - &["hash-object", "-w", "--stdin", "--no-filters"], - &image.bytes, - ) - .await?, - ) - .map_err(|_| "Invalid raw Git blob identity")? - .trim() - .to_owned(); + let object = match blobs.get(image.bytes.as_slice()) { + Some(object) => object.clone(), + None => { + let object = String::from_utf8( + self.input( + &["hash-object", "-w", "--stdin", "--no-filters"], + &image.bytes, + ) + .await?, + ) + .map_err(|_| "Invalid raw Git blob identity")? + .trim() + .to_owned(); + blobs.insert(image.bytes.as_slice(), object.clone()); + object + } + }; images::verify(&object, &image.bytes)?; let (parent, leaf) = path.rsplit_once('/').unwrap_or(("", path)); let mut ancestor = parent; @@ -79,3 +89,6 @@ impl Shadow { Ok(object) } } + +#[cfg(test)] +mod tests; diff --git a/crates/cyber-snapshot/src/git/raw/tests.rs b/crates/cyber-snapshot/src/git/raw/tests.rs new file mode 100644 index 000000000..fca7a2896 --- /dev/null +++ b/crates/cyber-snapshot/src/git/raw/tests.rs @@ -0,0 +1,136 @@ +//! Real Git process accounting, isolated from other tests' PATH and environment. +use super::*; +use std::os::unix::fs::PermissionsExt; +use std::process::Command; + +#[test] +fn repeated_raw_capture_images_write_each_distinct_blob_once_per_capture() { + let temp = tempfile::tempdir().unwrap(); + let bin = temp.path().join("bin"); + std::fs::create_dir(&bin).unwrap(); + let wrapper = bin.join("git"); + std::fs::write( + &wrapper, + r#"#!/bin/sh +for argument in "$@"; do + if [ "$argument" = hash-object ]; then + printf 'blob\n' >> "$CYBER_RAW_BLOB_CALLS" + fi +done +exec "$CYBER_RAW_REAL_GIT" "$@" +"#, + ) + .unwrap(); + std::fs::set_permissions(wrapper, std::fs::Permissions::from_mode(0o700)).unwrap(); + let real = Command::new("which").arg("git").output().unwrap(); + assert!(real.status.success()); + let real = std::str::from_utf8(&real.stdout).unwrap().trim(); + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "git::raw::tests::raw_blob_process_child", + "--nocapture", + ]) + .env("CYBER_RAW_REAL_GIT", real) + .env("CYBER_RAW_BLOB_CALLS", temp.path().join("calls")) + .env( + "PATH", + format!( + "{}:{}", + bin.display(), + std::env::var("PATH").unwrap_or_default() + ), + ) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test] +#[ignore = "subprocess fixture with isolated Git process accounting"] +async fn raw_blob_process_child() { + for format in ["sha1", "sha256"] { + verify_format(format).await; + } +} + +async fn verify_format(format: &str) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let repo = root.join("repo"); + std::fs::create_dir(&repo).unwrap(); + let status = Command::new(std::env::var_os("CYBER_RAW_REAL_GIT").unwrap()) + .args([ + "init", + "--quiet", + "-b", + "main", + &format!("--object-format={format}"), + ]) + .current_dir(&repo) + .status() + .unwrap(); + assert!(status.success()); + let shadow = Shadow::locate(&root.join("snapshots"), &repo).unwrap(); + shadow.ensure_init().await.unwrap(); + let mut files = raw_files(); + let calls = std::env::var_os("CYBER_RAW_BLOB_CALLS").unwrap(); + std::fs::write(&calls, "").unwrap(); + let first = shadow.write_raw_capture(&files).await.unwrap(); + assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 2); + verify_graph(&shadow, &first, &files).await; + assert_eq!(shadow.write_raw_capture(&files).await.unwrap(), first); + assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 4); + files.get_mut("nested/file_01").unwrap().bytes = b"later bytes".to_vec(); + let changed = shadow.write_raw_capture(&files).await.unwrap(); + assert_ne!(changed, first); + assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 7); +} + +fn raw_files() -> BTreeMap { + let mut files = (0..64) + .map(|index| { + ( + format!("nested/file_{index:02}"), + RawImage { + mode: if index == 0 { "100755" } else { "100644" }, + bytes: b"raw\0binary\r\n".to_vec(), + }, + ) + }) + .collect::>(); + files.insert( + "literal\nlink".into(), + RawImage { + mode: "120000", + bytes: b"nested/file_00".to_vec(), + }, + ); + files.get_mut("nested/file_63").unwrap().bytes = b"nested/file_00".to_vec(); + files +} + +async fn verify_graph(shadow: &Shadow, tree: &str, files: &BTreeMap) { + let catalog = shadow.read_catalog(tree.to_owned(), &[]).await.unwrap(); + assert_eq!(catalog.len(), 65); + assert_eq!(catalog["nested/file_00"].0, "100755"); + assert_eq!(catalog["nested/file_01"].0, "100644"); + assert_eq!(catalog["literal\nlink"].0, "120000"); + assert_eq!(catalog["nested/file_00"].1, catalog["nested/file_01"].1); + assert_eq!(catalog["nested/file_63"].0, "100644"); + assert_eq!(catalog["nested/file_63"].1, catalog["literal\nlink"].1); + for (path, (_, object)) in &catalog { + let bytes = shadow + .run_bounded(&["cat-file", "blob", object], 1024) + .await + .unwrap(); + assert_eq!(bytes, files[path].bytes); + images::verify(object, &bytes).unwrap(); + } +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 97aefebb1..7220d9692 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -151,6 +151,15 @@ fixture is retained/uploaded; independent client/JSONL/structured checks pass. That acceptance failure remains unresolved. The queued writer run at `5be366b7` and new native retirement need their own evidence. +The retained Intel fixture shows a 40,253 ms preview and no published preparation +origin before the later confirmation observation failure. Raw Git publication +performed one blob-writing process per path, even for identical captured bytes. +It now writes each distinct content once within that capture while retaining every +path/mode and existing source/policy/hash checks. The real managed cancellation +case passes locally with its origin observed in 14.6 seconds under the unchanged +45-second deadline. This is a local correction and diagnostic improvement; fresh +Intel/native acceptance remains required before resolving the platform failure. + Verification: three structured-output CLI tests, seven JSONL CLI tests, five structured-result server tests and six binary-worker ownership tests pass; formatting and whitespace diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index cb6fecec4..95383dc2f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -625,6 +625,12 @@ silently promote canonical blobs into raw-state evidence. This does not admit ordinary Git workers or resolve their lifetime, acknowledgement, configuration race, reclamation or complete compatibility/acceptance gates. +Raw graph publication MAY reuse one written Git blob for exactly equal immutable +bytes within one capture. Every path and mode SHALL retain its own tree entry; +source observation, complete-capture revalidation and per-placement image bounds +remain mandatory. The write cache SHALL NOT survive the capture or confer later +object integrity, source admission or native recovery authority. + Managed Git preview SHALL use the same native raw-image planner as restore, including expected-state/policy revalidation, historical exclusions, allowed paths and all-plan conflict refusal. Preview SHALL publish no journal or source effects. diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 5f03e9b9e..bfe0dec31 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1114,3 +1114,42 @@ has a test gap: native preparation/paired retirement needs direct cases, and man retirement remains unsupported. These are required before accepting M6/delivery. Full workspace, SDK runtime, current-source CLI/TUI and cross-platform acceptance were not rerun for this checkpoint. No PR, merge or parent completion is claimed. + +Intel cancellation investigation (2026-11-10): downloaded the retained failed +fixture from native run 42759434324, artifact 13741826409. Its original archive +remains untouched. A separate read-only SQLite inspection finds no published +snapshot preparation origin; server logs show the preceding rewind preview took +40,253 ms. This narrows the failure to the pre-origin request path but does not +prove the final cause or acceptance. The confirmation client previously discarded +its output when the observation assertion failed; its stdout/stderr now remain +inside the retained fixture, with exit status and observed-origin timing in logs. +The existing 45-second origin, 120-second staging and native pre-effect assertions +are unchanged. + +Raw graph publication was starting one hash-object process per path even for +identical immutable captured bytes. The new isolated real-Git regression initially +fails with 65 calls for two contents. An exact-byte borrowed map now writes each +distinct blob once per capture while preserving every path/mode/tree record and +Git hash verification. The cache never crosses captures or source observations. +The regression passes for SHA-1 and SHA-256, binary bytes, literal newline paths, +executable/regular/link entries, equal bytes across different modes, repeated +capture and changed content (2/4/7 cumulative writes). It does not replace source +identity/policy capture checks or per-placement handoff quota accounting. + +Final local raw-publication verification: the real managed Git prepared-cancellation +case passes in 130.51 seconds with origin observation at 14.5746 seconds. All original +45/120-second deadlines and actual source-admission/cancellation/payload assertions +remain. The feature snapshot package passes 154 library, 31 file-copy and 22 Git +cases; nine library subprocess fixtures and one Git subprocess fixture are ignored +at top level. The new process-accounting child actually runs both formats through +its parent. Windows-only/doc binaries select zero on this host. Snapshot/CLI all-target Clippy passes with warnings denied +following a test-helper split (the initial helper exceeded 15); no lint allowance +was added. Formatting, whitespace, strict OpenSpec (100 items) and spec lint +(0 errors/25 warnings) pass. Native C sandbox deprecations and a linker unwind-size +warning were observed without assigning their origin. Code/security review finds +no additional issue in this exact-byte, operation-local optimization: no source +observation, path/mode placement, hash check or quota proof is removed. Native CI +path selection now also covers App snapshot support fixtures and nested Store +reclamation protocol files. Full branch review, workspace/SDK runtime, other +CLI/TUI scenarios and fresh source-bound platform acceptance remain required; +this does not complete the Intel acceptance failure, M6 or issue #93. From b42f5fcaa801b456a2a2a7843fca9d5fc91eeabf Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 03:22:06 -0400 Subject: [PATCH 66/87] Verify metadata retirement across cancellation and paired recovery --- .github/workflows/non-git-snapshot-proof.yml | 20 ++++ crates/cyber-cli/tests/branches_cli.rs | 91 +++++++++++++++++++ crates/cyber-cli/tests/support/reclamation.rs | 6 +- .../src/copy/reclamation/retirement.rs | 4 +- docs/goal-delivery.md | 8 +- .../complete-daily-coding-surfaces/tasks.md | 39 ++++++++ 6 files changed, 164 insertions(+), 4 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 43aad5084..5a6c77cef 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -104,6 +104,24 @@ jobs: set -euo pipefail mkdir -p "$TMPDIR" cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli preparation_ -- --nocapture --test-threads=1 2>&1 | tee preparation-client-proof.log + - name: Prepared cancellation metadata retirement + if: ${{ !cancelled() }} + env: + TMPDIR: ${{ runner.temp }}/snapshot-fixtures + shell: bash + run: | + set -euo pipefail + mkdir -p "$TMPDIR" + cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli fully_prepared_pre_effect_cancellation_reclaims_only_persisted_target_inodes -- --exact --nocapture --test-threads=1 2>&1 | tee preparation-retirement-proof.log + - name: Paired metadata retirement after native source kill + if: ${{ !cancelled() }} + env: + TMPDIR: ${{ runner.temp }}/snapshot-fixtures + shell: bash + run: | + set -euo pipefail + mkdir -p "$TMPDIR" + cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_keep_and_rollback_after_real_partial_restore_kill_preserve_owner_files_and_never_replay -- --exact --nocapture --test-threads=1 2>&1 | tee paired-retirement-proof.log - name: JSONL driver and atomic last-message regressions if: ${{ !cancelled() }} shell: bash @@ -130,6 +148,8 @@ jobs: fallback-collection-proof.log git-cli-recovery-proof.log preparation-client-proof.log + preparation-retirement-proof.log + paired-retirement-proof.log jsonl-driver-proof.log structured-result-proof.log if-no-files-found: warn diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 105b3b31a..089ba8de3 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -1431,6 +1431,62 @@ fn assert_prepared_payload_cleanup(env: &Env, reply: &Value) { std::fs::read_to_string(env.root.join("preparation-owner-target")).unwrap(), "later owner bytes" ); + assert_prepared_metadata_retirement(&maintenance, &source, &journal, &receipt); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_prepared_metadata_retirement( + maintenance: &reclamation_support::Maintenance, + source: &std::path::Path, + journal: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + let owner = receipt.owners().unwrap().remove(0); + if owner.lifetime.owner.worktree_id.is_some() { + let error = maintenance.retire(source, receipt.clone()).unwrap_err(); + assert!( + error.contains("Managed checkout metadata retirement is not yet available"), + "{error}" + ); + assert!( + maintenance + .policy + .record(receipt) + .unwrap() + .unwrap() + .metadata + .is_none() + ); + assert!(journal.join("intent.json").exists() && journal.join("preparation.json").exists()); + return; + } + assert!(maintenance.retire(source, receipt.clone()).unwrap() > 0); + let record = maintenance.policy.record(receipt).unwrap().unwrap(); + assert!(record.completed && record.retired); + assert_eq!(record.metadata.unwrap().files.len(), 5); + assert!(!journal.exists()); + assert_retired_worker_files(&maintenance.namespace, receipt); + assert_eq!(maintenance.retire(source, receipt.clone()).unwrap(), 0); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_retired_worker_files( + namespace: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + for owner in receipt.owners().unwrap() { + for name in [ + owner.lifetime.owner.activity_id.as_str(), + owner.lifetime.source_phase.as_deref().unwrap(), + ] { + let error = namespace + .join("workers") + .join(name) + .symlink_metadata() + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::NotFound); + } + } } #[cfg(all( @@ -2276,6 +2332,41 @@ fn assert_paired_payload_cleanup(env: &Env, result: &Value) { std::fs::read_to_string(target).unwrap(), "later owner target" ); + assert_paired_metadata_retirement(&runtime, &snapshots, &policy, &receipt, &journals); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_paired_metadata_retirement( + runtime: &tokio::runtime::Runtime, + snapshots: &cyber_snapshot::GitSnapshots, + policy: &std::sync::Arc, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, + journals: &[PathBuf; 2], +) { + let source = receipt + .origin() + .unwrap() + .binding + .directory + .to_str() + .unwrap(); + assert!( + runtime + .block_on(snapshots.retire(source, receipt.clone())) + .unwrap() + > 0 + ); + let record = policy.record(receipt).unwrap().unwrap(); + assert!(record.completed && record.retired); + assert_eq!(record.metadata.unwrap().files.len(), 8); + assert!(journals.iter().all(|journal| !journal.exists())); + assert_retired_worker_files(journals[0].parent().unwrap().parent().unwrap(), receipt); + assert_eq!( + runtime + .block_on(snapshots.retire(source, receipt.clone())) + .unwrap(), + 0 + ); } #[cfg(any(target_os = "macos", target_os = "linux"))] diff --git a/crates/cyber-cli/tests/support/reclamation.rs b/crates/cyber-cli/tests/support/reclamation.rs index 713d5fd21..494196cc3 100644 --- a/crates/cyber-cli/tests/support/reclamation.rs +++ b/crates/cyber-cli/tests/support/reclamation.rs @@ -1,4 +1,4 @@ -//! Stop after a durable plan; native payload deletion is deliberately not run. +//! Real Store/native maintenance and a plan-only wrapper for interruption tests. use cyber_server::runtime::{ SnapshotOrigin, snapshot_reclamation::{Plan, Receipt, Record}, @@ -92,6 +92,10 @@ impl Maintenance { self.runtime .block_on(self.snapshots.reclaim(source.to_str().unwrap())) } + pub fn retire(&self, source: &Path, receipt: Receipt) -> Result { + self.runtime + .block_on(self.snapshots.retire(source.to_str().unwrap(), receipt)) + } pub fn plan_only(&self, source: &Path) -> Result { let cut = Self::snapshots( &self.root, diff --git a/crates/cyber-snapshot/src/copy/reclamation/retirement.rs b/crates/cyber-snapshot/src/copy/reclamation/retirement.rs index 0a8e87d62..61fc24127 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/retirement.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/retirement.rs @@ -30,7 +30,9 @@ pub(crate) fn retire( return Ok(0); } for owner in receipt.owners().map_err(storage::fail)? { - if owner.lifetime.checkout_resources_digest(&[])? != owner.resources_digest { + if owner.lifetime.owner.worktree_id.is_some() + || owner.lifetime.checkout_resources_digest(&[])? != owner.resources_digest + { return Err("Managed checkout metadata retirement is not yet available".into()); } } diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 7220d9692..ea41deaeb 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -118,12 +118,16 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. active-intent deletion, including a tested real process kill; cap pressure can reclaim metadata before one capture retry. Completed acknowledgement returns the retained Store receipt without reopening deleted files. Managed claims, - partial construction and complete receipt-kind/platform acceptance remain open. + partial construction and complete receipt-kind crash/platform acceptance remain + open. Real prepared-cancellation and paired keep/rollback CLI fixtures now + exercise metadata retirement and receipt acknowledgement after restart, including + no-follow checks for removed lifetime/phase files. These positive cases do not + establish the complete crash matrix or managed claim retirement. A separate real App regression reproduced 57,344 bytes of payload deletion when configuration resolved to null. Cleanup now preserves bytes initially and stops after policy loss between unlinks, retaining its exact resumable plan. - Thirty-seven App reclamation tests pass locally, including actual native + At `053dbfac`, 37 App reclamation tests pass locally, including actual native deletion, tamper/alias/live-owner refusals, policy loss, process death and cap retry. The earlier 14 migration tests cover the writer catalog. Fresh platform and broader current-source verification remain required before M6 acceptance. diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index bfe0dec31..ba3ee1590 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1153,3 +1153,42 @@ path selection now also covers App snapshot support fixtures and nested Store reclamation protocol files. Full branch review, workspace/SDK runtime, other CLI/TUI scenarios and fresh source-bound platform acceptance remain required; this does not complete the Intel acceptance failure, M6 or issue #93. + +Native receipt-kind retirement continuation (2026-11-10): existing actual CLI +preparation and partial-source-kill keep/rollback fixtures now invoke metadata +retirement only after observed payload completion. Prepared cancellation retains +its immutable five-file plan; paired recovery retains eight semantic leaves for +two exact owners. Tests require journal removal, no-follow absence of each lifetime +and source-phase path, observed completion, zero-effect repeat retirement, and the +original receipt through restarted real CLI acknowledgement. Existing later-owner +symlink/bytes, transcript, model-request and source-result assertions remain. +The native matrix now selects these two full cases in independent steps even after +another step fails, retains separate logs and preserves failed fixture directories. + +The new managed-preparation refusal assertion initially fails on the error text: +empty-set digest validation returned a generic incomplete-resource error before +the adapter's intended unsupported-managed diagnostic. Its local fixture remains +at .tmpskmwRM. The adapter now explicitly refuses managed ownership before that +empty-set calculation; complete receipt verification and every native/Store fence +remain mandatory. The regression requires the specific unavailable result, no +metadata plan, and retained intent/cancellation files. This grants no managed +retirement authority. Managed claims, zero-payload and partial construction, +old/deleted Location maintenance and complete crash/platform acceptance remain +required before M6 or its parent tasks can close. + +Final receipt-kind checks pass: all three exact CLI cases run serially and pass +with final no-follow worker-file absence assertions and the explicit managed +refusal. The cases cover prepared cancellation, managed Git cancellation (retained +unsupported metadata) and both actual partial-kill keep/rollback actions. The +corrected run reports 220.47 seconds, including an early fixture pause to let the +previous run finish; that elapsed time is not a performance benchmark. Its native +origin observations occur after resumption under the original deadline. The prior +run's two positives and one diagnostic mismatch are retained as failures rather +than counted as a green suite. Snapshot/CLI all-target Clippy with warnings denied, +formatting/whitespace, strict OpenSpec (100 items), spec lint (0 errors/25 warnings) +and Ruby YAML parsing pass. Code-review-skill review finds no additional issue in +the typed receipt-based assertions or the refusal-only guard correction. No new +HTTP/generated contract changes. Full workspace, SDK runtime, other CLI/TUI cases, +all receipt-kind crash cuts and new-source platform acceptance remain unperformed +here and required by their parent tasks. No M6, issue, PR or delivery completion is +claimed. The preserved original pre-main-integration stash remains intact. From 991b1519fb26ef80b486464df6edcffb07c62367 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 04:37:56 -0400 Subject: [PATCH 67/87] Retire consumed checkout claims through durable native metadata plans --- .github/workflows/non-git-snapshot-proof.yml | 13 + Cargo.lock | 1 + .../cyber-app/tests/snapshot_git_runtime.rs | 125 ++++- .../support/snapshot_native_retirement.rs | 27 ++ .../tests/support/snapshot_retirement.rs | 1 + crates/cyber-cli/tests/branches_cli.rs | 110 ++++- crates/cyber-core/Cargo.toml | 1 + crates/cyber-core/src/worktrees.rs | 3 + crates/cyber-core/src/worktrees/activity.rs | 3 + .../src/worktrees/activity/snapshot.rs | 18 +- .../activity/snapshot/acknowledgement.rs | 31 ++ .../worktrees/activity/snapshot/retirement.rs | 197 ++++++++ .../activity/snapshot/retirement/native.rs | 445 ++++++++++++++++++ .../snapshot/retirement/native/tests.rs | 310 ++++++++++++ .../runtime/snapshot_reclamation/metadata.rs | 110 ++++- .../src/copy/reclamation/checkouts.rs | 3 + .../src/copy/reclamation/retirement.rs | 19 +- .../src/copy/reclamation/retirement/native.rs | 74 ++- ...110010000_snapshot_checkout_retirement.sql | 3 + crates/cyber-store/src/migrate.rs | 14 +- docs/goal-delivery.md | 21 +- .../managed-recovery.md | 27 ++ .../complete-daily-coding-surfaces/tasks.md | 91 ++++ 23 files changed, 1580 insertions(+), 67 deletions(-) create mode 100644 crates/cyber-core/src/worktrees/activity/snapshot/retirement.rs create mode 100644 crates/cyber-core/src/worktrees/activity/snapshot/retirement/native.rs create mode 100644 crates/cyber-core/src/worktrees/activity/snapshot/retirement/native/tests.rs create mode 100644 crates/cyber-store/migrations/20261110010000_snapshot_checkout_retirement.sql diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 5a6c77cef..8964deaba 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -77,11 +77,23 @@ jobs: set -euo pipefail cargo test --locked -p cyber-core --lib worktrees::activity::snapshot::acknowledgement::tests::maintenance_requires_an_existing_acknowledgement_without_publishing_one -- --exact --nocapture 2>&1 | tee managed-checkout-ack-proof.log - name: Owned Git runtime acceptance candidate + env: + TMPDIR: ${{ runner.temp }}/snapshot-fixtures shell: bash run: | set -euo pipefail + mkdir -p "$TMPDIR" cargo test --locked -p cyber-snapshot --features native-git-acceptance -- --nocapture 2>&1 | tee git-adapter-proof.log cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_git_runtime -- --nocapture 2>&1 | tee git-runtime-proof.log + - name: Exact acknowledged checkout retirement and resumption + if: ${{ !cancelled() }} + env: + TMPDIR: ${{ runner.temp }}/snapshot-fixtures + shell: bash + run: | + set -euo pipefail + mkdir -p "$TMPDIR" + cargo test --locked -p cyber-core --lib worktrees::activity::snapshot::retirement:: -- --nocapture 2>&1 | tee checkout-retirement-proof.log - name: Store-bound fallback collection and reclamation shell: bash run: | @@ -143,6 +155,7 @@ jobs: path: | non-git-snapshot-proof.log managed-checkout-ack-proof.log + checkout-retirement-proof.log git-adapter-proof.log git-runtime-proof.log fallback-collection-proof.log diff --git a/Cargo.lock b/Cargo.lock index 1da8d909f..71ef6f8e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1153,6 +1153,7 @@ name = "cyber-core" version = "0.1.0" dependencies = [ "aho-corasick", + "blake3", "cap-fs-ext", "cap-std", "chrono", diff --git a/crates/cyber-app/tests/snapshot_git_runtime.rs b/crates/cyber-app/tests/snapshot_git_runtime.rs index 2834c82ad..7f9ddc756 100644 --- a/crates/cyber-app/tests/snapshot_git_runtime.rs +++ b/crates/cyber-app/tests/snapshot_git_runtime.rs @@ -26,6 +26,18 @@ struct Fixture { directory: PathBuf, session: String, } +impl Drop for Fixture { + fn drop(&mut self) { + if std::thread::panicking() { + self._temp.disable_cleanup(true); + eprintln!( + "Preserved failed Git App fixture at {}", + self._temp.path().display() + ); + } + } +} + impl Fixture { async fn new(format: &str, scoped: bool) -> Self { let temp = tempfile::tempdir().unwrap(); @@ -347,13 +359,122 @@ async fn reclaim_restore_payloads(f: &Fixture, receipt: &SnapshotRestoreReceipt) .unwrap(), 0 ); + assert_retained_intent_metadata(&journal, &intent); + retire_restore_metadata(f, receipt, &snapshots, policy.as_ref()).await; + assert_reclaimed_source_and_receipt(f, receipt, inference).await; +} + +fn assert_retained_intent_metadata(journal: &Path, intent: &[u8]) { assert_eq!(std::fs::read(journal.join("intent.json")).unwrap(), intent); - let header: Value = serde_json::from_slice(&intent).unwrap(); + let header: Value = serde_json::from_slice(intent).unwrap(); assert_eq!(header["version"], 7); assert!(header["updates"].is_object()); assert!(journal.join("intent.next").is_file()); assert_eq!(std::fs::read_dir(journal).unwrap().count(), 2); - assert_reclaimed_source_and_receipt(f, receipt, inference).await; +} + +async fn retire_restore_metadata( + f: &Fixture, + receipt: &SnapshotRestoreReceipt, + snapshots: &cyber_snapshot::GitSnapshots, + policy: &dyn cyber_snapshot::reclamation::Policy, +) { + use cyber_server::runtime::snapshot_reclamation::Receipt; + let journal = f.journal(receipt); + let namespace = journal.parent().unwrap().parent().unwrap().to_path_buf(); + let checkout = receipt.checkout.as_ref().unwrap(); + let phase = namespace + .join("workers") + .join(checkout.lifetime.source_phase.as_ref().unwrap()); + let data: Value = serde_json::from_slice(&std::fs::read(&phase).unwrap()).unwrap(); + let bindings: Vec = + serde_json::from_value(data["checkout_bindings"].clone()).unwrap(); + let creations = bindings + .iter() + .map(|binding| { + let path = binding + .common_dir + .join("cyber-worktrees") + .join(format!("{}.json", binding.name)); + let bytes = std::fs::read(&path).unwrap(); + (path, bytes) + }) + .collect::>(); + let retained = Receipt::Restore(receipt.clone()); + assert!( + snapshots + .retire(f.directory.to_str().unwrap(), retained.clone()) + .await + .unwrap() + > 0 + ); + let record = policy.record(&retained).unwrap().unwrap(); + assert!(record.completed && record.retired); + let plan = record.metadata.unwrap(); + assert_managed_retirement_frame(&plan, &record.plan); + assert_eq!( + plan.checkouts + .iter() + .map(|plan| plan.binding.clone()) + .collect::>(), + bindings + ); + assert!(!journal.exists()); + assert_eq!( + std::fs::symlink_metadata(phase).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + for binding in &bindings { + assert_checkout_metadata_removed(binding); + } + for (path, bytes) in creations { + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + assert_eq!( + snapshots + .retire(f.directory.to_str().unwrap(), retained) + .await + .unwrap(), + 0 + ); +} +fn assert_managed_retirement_frame( + plan: &cyber_server::runtime::snapshot_reclamation::metadata::Plan, + payload: &cyber_server::runtime::snapshot_reclamation::Plan, +) { + plan.validate(payload).unwrap(); + if plan.checkouts.is_empty() { + return; + } + for mutation in ["receipt", "resources", "owner", "duplicate", "alias"] { + let mut foreign = plan.clone(); + match mutation { + "receipt" => foreign.checkouts[0].receipt_digest = "f".repeat(64), + "resources" => foreign.checkouts[0].resources_digest = "f".repeat(64), + "owner" => foreign.checkouts[0].binding.owner_id = "op_foreign".into(), + "duplicate" => foreign.checkouts.push(foreign.checkouts[0].clone()), + "alias" => { + foreign.checkouts[0].acknowledgement.identity = + foreign.checkouts[0].claim.identity.clone() + } + _ => unreachable!(), + } + assert!(foreign.validate(payload).is_err(), "{mutation}"); + } +} + +fn assert_checkout_metadata_removed(binding: &cyber_core::worktrees::CheckoutBinding) { + let directory = binding + .common_dir + .join("cyber-worktree-activity") + .join(&binding.worktree_id); + for suffix in [".lock", ".snapshot-ack"] { + let path = directory.join(format!("{}{suffix}", binding.owner_id)); + assert_eq!( + std::fs::symlink_metadata(path).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + } } async fn assert_replaced_link_payload_refused( diff --git a/crates/cyber-app/tests/support/snapshot_native_retirement.rs b/crates/cyber-app/tests/support/snapshot_native_retirement.rs index 6c99cb1f5..3b817f179 100644 --- a/crates/cyber-app/tests/support/snapshot_native_retirement.rs +++ b/crates/cyber-app/tests/support/snapshot_native_retirement.rs @@ -328,6 +328,33 @@ async fn native_retirement_respects_retention_and_reader_leases_before_planning( f.app.runtime.shutdown().await; } +#[tokio::test] +async fn native_retirement_preserves_changed_intent_and_admission_context_before_planning() { + for mode in ["intent", "phase"] { + let f = Fixture::new().await; + payload_complete(&f).await; + let path = if mode == "intent" { + f.journal.join("intent.json") + } else { + worker_routes(&f)[1].clone() + }; + let mut value: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + value[if mode == "intent" { + "acknowledged" + } else { + "admitted" + }] = false.into(); + let changed = serde_json::to_vec(&value).unwrap(); + std::fs::write(&path, &changed).unwrap(); + assert!(retire(&f).await.is_err(), "{mode}"); + assert!(f.record().unwrap().metadata.is_none()); + assert_eq!(std::fs::read(path).unwrap(), changed); + assert!(f.journal.join("intent.json").exists()); + assert!(worker_routes(&f).iter().all(|path| path.exists())); + f.app.runtime.shutdown().await; + } +} + #[tokio::test] async fn native_metadata_retirement_child() { let Ok(control) = std::env::var("CYBER_NATIVE_METADATA_RETIREMENT_CHILD") else { diff --git a/crates/cyber-app/tests/support/snapshot_retirement.rs b/crates/cyber-app/tests/support/snapshot_retirement.rs index 2c25b160c..55aa4a0cc 100644 --- a/crates/cyber-app/tests/support/snapshot_retirement.rs +++ b/crates/cyber-app/tests/support/snapshot_retirement.rs @@ -66,6 +66,7 @@ fn plan(f: &Fixture) -> Plan { version: 1, payload_plan_id: records::Plan::id(&f.receipt).unwrap(), files, + checkouts: vec![], } } fn append( diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index 089ba8de3..b13fb1276 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -926,6 +926,7 @@ fn managed_git_partial_kill(action: &str) { assert_native_settlement_link(&env, &journal, &result); assert_recovery_lookup_does_not_replay(&env, &id, &request_id, &request, &result, &messages); assert_managed_git_reclamation(&env, &result); + assert_retired_recovery_receipt(&env, &id, &request_id, &result, &messages); assert_eq!(std::fs::read(index_path).unwrap(), index); assert_eq!(std::fs::read(config_path).unwrap(), config); assert_eq!(model.requests().len(), requests); @@ -993,6 +994,74 @@ fn assert_managed_git_reclamation(env: &Env, result: &Value) { source_link ); assert!(marker.exists()); + assert_managed_metadata_retired(&maintenance, &env.directory, &receipt); +} + +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn assert_retired_recovery_receipt( + env: &Env, + session: &str, + request: &str, + expected: &Value, + messages: &[Value], +) { + let source = env.file("bulk_0000"); + let lookup = json(&env.cyber(&[ + "sessions", + "recovery-request", + session, + request, + "--format", + "json", + ])); + assert_eq!(lookup["receipt"], expected["receipt"]); + assert!(lookup["journal_acknowledged"].is_null()); + kill_registered_service(env); + let acknowledged = json(&env.cyber(&[ + "sessions", + "recovery-request", + session, + request, + "--acknowledge", + "--format", + "json", + ])); + assert_eq!(acknowledged["receipt"], expected["receipt"]); + assert_eq!(acknowledged["journal_acknowledged"], true); + assert_eq!(env.file("bulk_0000"), source); + assert_eq!(env.messages(session).as_slice(), messages); +} + +#[cfg(all( + feature = "native-git-acceptance", + any(target_os = "macos", target_os = "linux") +))] +fn assert_managed_metadata_retired( + maintenance: &reclamation_support::Maintenance, + source: &std::path::Path, + receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, +) { + assert!(maintenance.retire(source, receipt.clone()).unwrap() > 0); + let record = maintenance.policy.record(receipt).unwrap().unwrap(); + assert!(record.completed && record.retired); + let metadata = record.metadata.unwrap(); + assert_eq!(metadata.version, 2); + assert!(!metadata.checkouts.is_empty()); + assert_retired_checkout_files(&metadata.checkouts); + for owner in receipt.owners().unwrap() { + assert!( + !maintenance + .namespace + .join("journals") + .join(&owner.origin.binding.journal_id) + .exists() + ); + } + assert_retired_worker_files(&maintenance.namespace, receipt); + assert_eq!(maintenance.retire(source, receipt.clone()).unwrap(), 0); } #[cfg(all( @@ -1441,34 +1510,35 @@ fn assert_prepared_metadata_retirement( journal: &std::path::Path, receipt: &cyber_server::runtime::snapshot_reclamation::Receipt, ) { - let owner = receipt.owners().unwrap().remove(0); - if owner.lifetime.owner.worktree_id.is_some() { - let error = maintenance.retire(source, receipt.clone()).unwrap_err(); - assert!( - error.contains("Managed checkout metadata retirement is not yet available"), - "{error}" - ); - assert!( - maintenance - .policy - .record(receipt) - .unwrap() - .unwrap() - .metadata - .is_none() - ); - assert!(journal.join("intent.json").exists() && journal.join("preparation.json").exists()); - return; - } assert!(maintenance.retire(source, receipt.clone()).unwrap() > 0); let record = maintenance.policy.record(receipt).unwrap().unwrap(); assert!(record.completed && record.retired); - assert_eq!(record.metadata.unwrap().files.len(), 5); + let metadata = record.metadata.unwrap(); + assert_eq!(metadata.files.len(), 5); + assert_retired_checkout_files(&metadata.checkouts); assert!(!journal.exists()); assert_retired_worker_files(&maintenance.namespace, receipt); assert_eq!(maintenance.retire(source, receipt.clone()).unwrap(), 0); } +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn assert_retired_checkout_files(plans: &[cyber_core::worktrees::CheckoutRetirementPlan]) { + for plan in plans { + let directory = plan + .binding + .common_dir + .join("cyber-worktree-activity") + .join(&plan.binding.worktree_id); + for suffix in [".lock", ".snapshot-ack"] { + let path = directory.join(format!("{}{suffix}", plan.binding.owner_id)); + assert_eq!( + std::fs::symlink_metadata(path).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + } + } +} + #[cfg(any(target_os = "macos", target_os = "linux"))] fn assert_retired_worker_files( namespace: &std::path::Path, diff --git a/crates/cyber-core/Cargo.toml b/crates/cyber-core/Cargo.toml index 8d6f564b0..3f4dd0344 100644 --- a/crates/cyber-core/Cargo.toml +++ b/crates/cyber-core/Cargo.toml @@ -27,6 +27,7 @@ serde.workspace = true serde_json.workspace = true serde_yaml_ng = "0.10" sha2.workspace = true +blake3 = "1" shell-words = "1.1" similar = "3.2" unicode-normalization = "0.1" diff --git a/crates/cyber-core/src/worktrees.rs b/crates/cyber-core/src/worktrees.rs index de857ba43..4a85ad56f 100644 --- a/crates/cyber-core/src/worktrees.rs +++ b/crates/cyber-core/src/worktrees.rs @@ -8,10 +8,13 @@ mod location; mod removal; pub(crate) mod repository; mod setup; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub use activity::CheckoutRetirementGuard; pub use activity::{ CheckoutAcknowledgement, CheckoutActivity, CheckoutBinding, CheckoutLease, CheckoutOwner, SnapshotClaim, }; +pub use activity::{CheckoutRetirementFile, CheckoutRetirementPlan}; pub use changes::{ChangedFile, Changes}; pub use includes::IncludedFile; pub use removal::{RemovalActivity, RemovalPhase, RemovalRecord}; diff --git a/crates/cyber-core/src/worktrees/activity.rs b/crates/cyber-core/src/worktrees/activity.rs index 1ba168236..594ce52d5 100644 --- a/crates/cyber-core/src/worktrees/activity.rs +++ b/crates/cyber-core/src/worktrees/activity.rs @@ -8,7 +8,10 @@ use serde::{Deserialize, Serialize}; use super::{GitExecution, Managed, Name, RemovalActivity, Repository, RepositoryLock}; mod snapshot; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub use snapshot::CheckoutRetirementGuard; pub use snapshot::{CheckoutAcknowledgement, CheckoutBinding, CheckoutOwner, SnapshotClaim}; +pub use snapshot::{CheckoutRetirementFile, CheckoutRetirementPlan}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct UseRecord { diff --git a/crates/cyber-core/src/worktrees/activity/snapshot.rs b/crates/cyber-core/src/worktrees/activity/snapshot.rs index 57058a38b..9f6a30d8a 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot.rs @@ -2,10 +2,14 @@ use super::{Managed, UseRecord, invalid}; pub(super) mod acknowledgement; mod publication; +mod retirement; pub use acknowledgement::CheckoutAcknowledgement; use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, OpenOptions}; pub(super) use publication::publish; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub use retirement::CheckoutRetirementGuard; +pub use retirement::{CheckoutRetirementFile, CheckoutRetirementPlan}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ @@ -168,12 +172,14 @@ pub(super) struct Pins { } fn open_file(directory: &Dir, name: &str) -> io::Result { - let file = directory - .open_with( - name, - OpenOptions::new().read(true).follow(FollowSymlinks::No), - )? - .into_std(); + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + } + let file = directory.open_with(name, &options)?.into_std(); regular(&file)?; Ok(file) } diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs index e4155fa91..0ecaf6417 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs @@ -69,6 +69,18 @@ fn stage_owner(name: &str) -> io::Result<&str> { Ok(owner) } impl Acknowledged { + pub(super) fn retirement_context(&self) -> (String, String) { + ( + self.record.resources_digest.clone(), + self.record.receipt_digest.clone(), + ) + } + pub(super) fn into_retirement_file(self) -> LockedRecord { + self.file + } + pub(super) fn file_for_retirement(&self) -> &File { + &self.file + } fn open(pins: &Pins, binding: &CheckoutBinding) -> io::Result { let file = open_file(&pins.directory, &format!("{}{SUFFIX}", binding.owner_id))?; if !super::super::try_lock(&file)? { @@ -114,6 +126,25 @@ impl Acknowledged { } } +pub(super) fn verify_retirement_context( + file: &File, + plan: &super::CheckoutRetirementPlan, +) -> io::Result<()> { + let record: Record = serde_json::from_slice(&contents(file, LIMIT)?).map_err(invalid)?; + if record.version != 1 + || record.binding != plan.binding + || record.resources_digest != plan.resources_digest + || record.receipt_digest != plan.receipt_digest + || record.file_identity != plan.acknowledgement.identity + || stage_owner(&record.staging_name)? != plan.binding.owner_id + { + return Err(invalid( + "Checkout retirement acknowledgement context differs", + )); + } + Ok(()) +} + impl CheckoutAcknowledgement { pub fn new( binding: &CheckoutBinding, diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/retirement.rs b/crates/cyber-core/src/worktrees/activity/snapshot/retirement.rs new file mode 100644 index 000000000..b4ef1f730 --- /dev/null +++ b/crates/cyber-core/src/worktrees/activity/snapshot/retirement.rs @@ -0,0 +1,197 @@ +//! Frozen observations are data; deletion additionally needs the native guard +//! inside a consuming Store writer's reference/retention authorization callback. +use super::{CheckoutBinding, invalid}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeSet, io}; +const LIMIT: usize = 8192; +const TOTAL_BYTES: u64 = 64 * 1024 * 1024; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod native; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub use native::CheckoutRetirementGuard; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckoutRetirementFile { + pub identity: String, + pub generation: String, + pub hash: String, + pub bytes: u64, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckoutRetirementPlan { + pub version: u32, + pub binding: CheckoutBinding, + pub resources_digest: String, + pub receipt_digest: String, + pub repository_lock_identity: String, + pub claim: CheckoutRetirementFile, + pub acknowledgement: CheckoutRetirementFile, +} +pub(super) fn hash(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} +fn identity(value: &str) -> bool { + let mut parts = value.split(':'); + if value.len() > 256 { + return false; + } + for index in 0..7 { + let Some(part) = parts.next() else { + return false; + }; + let Ok(value) = part.parse::() else { + return false; + }; + if part.is_empty() + || !part.bytes().all(|byte| byte.is_ascii_digit()) + || index == 3 && value >= 1_000_000_000 + { + return false; + } + } + parts.next().is_none() +} +fn portable_absolute(path: &std::path::Path) -> bool { + let Some(path) = path.to_str().filter(|path| path.len() <= 4096) else { + return false; + }; + path.starts_with('/') + || path.starts_with("\\\\") + || path.as_bytes().get(1..3).is_some_and(|tail| { + tail[0] == b':' + && matches!(tail[1], b'/' | b'\\') + && path.as_bytes()[0].is_ascii_alphabetic() + }) +} +fn validate_binding(binding: &CheckoutBinding) -> io::Result<()> { + super::super::safe_id(&binding.owner_id, "op")?; + super::super::safe_id(&binding.worktree_id, "wt")?; + super::super::super::Name::parse(&binding.name).map_err(invalid)?; + let owner = binding + .lifetime + .as_ref() + .ok_or_else(|| invalid("Missing retirement lifetime"))?; + super::super::safe_id(&owner.session_id, "ses")?; + if binding.version != 2 + || owner.activity_id != binding.owner_id + || !hash(&owner.lifetime_digest) + || owner.session_created_id.is_empty() + || owner.session_created_id.len() > 256 + || !portable_absolute(&owner.directory) + || !portable_absolute(&binding.common_dir) + || !hash(&binding.creation_digest) + { + return Err(invalid("Invalid checkout retirement binding")); + } + validate_namespace(binding) +} +fn validate_namespace(binding: &CheckoutBinding) -> io::Result<()> { + for value in [ + &binding.common_identity, + &binding.root_identity, + &binding.directory_identity, + &binding.creation_directory_identity, + &binding.creation_file_identity, + ] { + if !identity(value) { + return Err(invalid("Invalid retirement namespace identity")); + } + } + let prefix = format!("preparing_{}_{}_", binding.owner_id.len(), binding.owner_id); + if binding.staging_name.as_deref().is_none_or(|stage| { + !stage.starts_with(&prefix) + || stage.len() != prefix.len() + 26 + || stage[prefix.len()..].parse::().is_err() + }) { + return Err(invalid("Invalid retirement construction binding")); + } + Ok(()) +} + +impl CheckoutRetirementFile { + pub fn validate(&self) -> io::Result<()> { + if !identity(&self.identity) + || !hash(&self.generation) + || !hash(&self.hash) + || self.bytes > 1024 * 1024 + { + return Err(invalid("Invalid checkout retirement file bounds")); + } + Ok(()) + } +} +impl CheckoutRetirementPlan { + /// Data validation only; never native acquisition, acknowledgement or deletion. + pub fn validate(&self) -> io::Result<()> { + let binding = &self.binding; + validate_binding(binding)?; + if self.version != 1 + || !hash(&self.resources_digest) + || !hash(&self.receipt_digest) + || !identity(&self.repository_lock_identity) + { + return Err(invalid("Invalid checkout retirement ownership")); + } + self.claim.validate()?; + self.acknowledgement.validate()?; + if self.claim.identity != binding.file_identity + || self.claim.identity == self.acknowledgement.identity + || self.repository_lock_identity == self.claim.identity + || self.repository_lock_identity == self.acknowledgement.identity + { + return Err(invalid("Aliased checkout retirement roles")); + } + Ok(()) + } + /// Bounds and uniqueness for a complete metadata frame; still no native authority. + pub fn validate_set(plans: &[Self]) -> io::Result<()> { + if plans.len() > LIMIT { + return Err(invalid("Checkout retirement resource limit exceeded")); + } + let mut owners = BTreeSet::new(); + let mut identities = BTreeSet::new(); + let mut bytes = 0u64; + for plan in plans { + plan.validate()?; + if !owners.insert(( + &plan.binding.common_dir, + &plan.binding.worktree_id, + &plan.binding.owner_id, + )) || !identities.insert(&plan.claim.identity) + || !identities.insert(&plan.acknowledgement.identity) + { + return Err(invalid( + "Duplicate checkout retirement resource or native file", + )); + } + bytes = bytes + .checked_add(plan.total_bytes()?) + .filter(|bytes| *bytes <= TOTAL_BYTES) + .ok_or_else(|| invalid("Checkout retirement total bytes exceed limit"))?; + } + Ok(()) + } + pub fn total_bytes(&self) -> io::Result { + self.claim + .bytes + .checked_add(self.acknowledgement.bytes) + .ok_or_else(|| invalid("Checkout retirement size overflow")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn historical_absolute_paths_validate_without_accessing_the_host_filesystem() { + for path in ["/old/deleted", r"C:\old\deleted", r"\\server\share\deleted"] { + assert!(portable_absolute(std::path::Path::new(path)), "{path}"); + } + for path in ["relative/path", "C:relative", "\\single-root"] { + assert!(!portable_absolute(std::path::Path::new(path)), "{path}"); + } + } +} diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/retirement/native.rs b/crates/cyber-core/src/worktrees/activity/snapshot/retirement/native.rs new file mode 100644 index 000000000..144d6d578 --- /dev/null +++ b/crates/cyber-core/src/worktrees/activity/snapshot/retirement/native.rs @@ -0,0 +1,445 @@ +//! Fixed native claim routes. The caller retains the consumed snapshot lifetime +//! and invokes each unlink only inside its durably planned Store writer callback. +use super::super::super::{LockedRecord, UseRecord}; +use super::super::{ + Pins, SnapshotClaim, acknowledgement, bytes, contents, digest, directory_identity, open_file, +}; +use super::*; +use cap_std::fs::Dir; +use std::os::unix::fs::MetadataExt; +use std::{ + collections::BTreeMap, + fs::File, + path::{Path, PathBuf}, +}; + +struct RepositoryPin { + path: PathBuf, + common_identity: String, + identity: String, + file: LockedRecord, +} +impl RepositoryPin { + fn open(binding: &CheckoutBinding, expected: Option<&str>) -> io::Result { + let common = common(&binding.common_dir, &binding.common_identity)?; + let file = open_file(&common, "cyber-worktree.lock")?; + if !super::super::super::try_lock(&file)? { + return Err(invalid("Checkout repository maintenance is busy")); + } + let file = LockedRecord(file); + let identity = super::super::identity(&file)?; + if expected.is_some_and(|expected| expected != identity) { + return Err(invalid("Checkout retirement repository lock changed")); + } + super::super::publication::sync_file(&file)?; + super::super::publication::sync_directory(&common)?; + let pin = Self { + path: binding.common_dir.clone(), + common_identity: binding.common_identity.clone(), + identity, + file, + }; + pin.verify()?; + Ok(pin) + } + fn verify(&self) -> io::Result<()> { + let common = common(&self.path, &self.common_identity)?; + let current = open_file(&common, "cyber-worktree.lock")?; + if super::super::identity(¤t)? != self.identity + || super::super::identity(&self.file)? != self.identity + { + return Err(invalid("Checkout retirement repository lock route changed")); + } + Ok(()) + } +} +fn common(path: &Path, expected: &str) -> io::Result { + if !path.is_absolute() { + return Err(invalid( + "Checkout retirement native path is unavailable on this host", + )); + } + if std::fs::symlink_metadata(path)?.file_type().is_symlink() { + return Err(invalid("Checkout retirement common route is a symlink")); + } + let dir = Dir::open_ambient_dir(path, cap_std::ambient_authority())?; + if directory_identity(&dir)? != expected { + return Err(invalid("Checkout retirement common route changed")); + } + Ok(dir) +} +fn generation(file: &File) -> io::Result { + let m = file.metadata()?; + Ok(digest( + &serde_json::to_vec(&( + m.dev(), + m.ino(), + m.mode(), + m.uid(), + m.gid(), + m.nlink(), + m.len(), + m.mtime(), + m.mtime_nsec(), + m.ctime(), + m.ctime_nsec(), + )) + .map_err(invalid)?, + )) +} +impl CheckoutRetirementFile { + fn observe(file: &File) -> io::Result { + super::super::regular(file)?; + let generation = generation(file)?; + let bytes = contents(file, 1024 * 1024)?; + let observed = Self { + identity: super::super::identity(file)?, + generation: generation.clone(), + hash: blake3::hash(&bytes).to_hex().to_string(), + bytes: bytes.len() as u64, + }; + observed.validate()?; + if generation != self::generation(file)? { + return Err(invalid("Checkout retirement file changed while hashing")); + } + Ok(observed) + } +} +struct Leaf { + file: LockedRecord, + present: bool, +} +impl Leaf { + fn open( + pins: &Pins, + name: &str, + expected: &CheckoutRetirementFile, + ) -> io::Result> { + let file = match open_file(&pins.directory, name) { + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + result => result?, + }; + if !super::super::super::try_lock(&file)? { + return Err(invalid("Checkout retirement leaf is still active")); + } + let file = LockedRecord(file); + if CheckoutRetirementFile::observe(&file)? != *expected { + return Err(invalid( + "Checkout retirement leaf differs from the frozen plan", + )); + } + Ok(Some(Self { + file, + present: true, + })) + } + fn verify(&self, pins: &Pins, name: &str, expected: &CheckoutRetirementFile) -> io::Result<()> { + if !self.present { + return absent(&pins.directory, name); + } + let current = open_file(&pins.directory, name)?; + super::super::regular(&self.file)?; + if super::super::identity(¤t)? != expected.identity + || super::super::identity(&self.file)? != expected.identity + || generation(¤t)? != expected.generation + || generation(&self.file)? != expected.generation + { + return Err(invalid( + "Checkout retirement leaf route or generation changed", + )); + } + Ok(()) + } +} +struct Claim { + plan: CheckoutRetirementPlan, + pins: Pins, + claim: Option, + acknowledgement: Option, +} +impl Claim { + fn capture(claim: SnapshotClaim, repository_lock_identity: String) -> io::Result { + let acknowledgement = claim + .acknowledgement + .as_ref() + .ok_or_else(|| invalid("Checkout retirement requires an existing acknowledgement"))?; + let (resources_digest, receipt_digest) = acknowledgement.retirement_context(); + let claim_file = CheckoutRetirementFile::observe(&claim.lease.file)?; + let acknowledgement_file = + CheckoutRetirementFile::observe(acknowledgement.file_for_retirement())?; + // Verify typed original context AFTER observing bytes/generations. The + // final guard check then rejects any mutation since those observations. + claim.verify()?; + let pins = claim + .lease + .snapshot_pins + .as_ref() + .ok_or_else(|| invalid("Missing checkout retirement pins"))?; + super::super::publication::sync_file(&claim.lease.file)?; + super::super::publication::sync_file(acknowledgement.file_for_retirement())?; + super::super::publication::sync_file(&pins.creation)?; + for directory in [&pins.common, &pins.root, &pins.directory, &pins.creations] { + super::super::publication::sync_directory(directory)?; + } + let SnapshotClaim { + lease, + acknowledgement, + } = claim; + let acknowledgement = + acknowledgement.ok_or_else(|| invalid("Missing captured checkout acknowledgement"))?; + let plan = CheckoutRetirementPlan { + version: 1, + binding: lease + .record + .snapshot_binding + .clone() + .ok_or_else(|| invalid("Missing checkout retirement binding"))?, + resources_digest, + receipt_digest, + repository_lock_identity, + claim: claim_file, + acknowledgement: acknowledgement_file, + }; + plan.validate()?; + Ok(Self { + plan, + pins: lease + .snapshot_pins + .ok_or_else(|| invalid("Missing checkout retirement pins"))?, + claim: Some(Leaf { + file: lease.file, + present: true, + }), + acknowledgement: Some(Leaf { + file: acknowledgement.into_retirement_file(), + present: true, + }), + }) + } + fn resume(plan: CheckoutRetirementPlan) -> io::Result { + plan.validate()?; + let pins = Pins::open( + &plan.binding.common_dir, + &plan.binding.worktree_id, + &plan.binding.name, + )?; + verify_creation(&pins, &plan.binding)?; + let mut claim = Leaf::open(&pins, &claim_name(&plan), &plan.claim)?; + if let Some(leaf) = &mut claim { + let record: UseRecord = super::super::super::read_record(&mut leaf.file)?; + if record.snapshot_binding.as_ref() != Some(&plan.binding) || record.settled { + return Err(invalid( + "Checkout retirement original claim context differs", + )); + } + pins.verify(&leaf.file, &record)?; + } + let acknowledgement = Leaf::open(&pins, &ack_name(&plan), &plan.acknowledgement)?; + if let Some(leaf) = &acknowledgement { + acknowledgement::verify_retirement_context(&leaf.file, &plan)?; + } + Ok(Self { + plan, + pins, + claim, + acknowledgement, + }) + } + fn verify(&self) -> io::Result<()> { + verify_creation(&self.pins, &self.plan.binding)?; + verify_leaf( + self.claim.as_ref(), + &self.pins, + &claim_name(&self.plan), + &self.plan.claim, + )?; + verify_leaf( + self.acknowledgement.as_ref(), + &self.pins, + &ack_name(&self.plan), + &self.plan.acknowledgement, + )?; + let owner = &self.plan.binding.owner_id; + let prefixes = [ + format!("preparing_{}_{owner}_", owner.len()), + format!("ack_preparing_{}_{owner}_", owner.len()), + ]; + for (index, entry) in self.pins.directory.entries()?.enumerate() { + let name = entry?.file_name(); + if index >= 4096 { + return Err(invalid("Checkout retirement inventory exceeds limit")); + } + if prefixes + .iter() + .any(|prefix| name.to_string_lossy().starts_with(prefix)) + { + return Err(invalid( + "Checkout retirement retains unknown same-owner construction evidence", + )); + } + } + Ok(()) + } +} +fn verify_creation(pins: &Pins, binding: &CheckoutBinding) -> io::Result<()> { + pins.matches(binding)?; + let current = Pins::open(&binding.common_dir, &binding.worktree_id, &binding.name)?; + current.matches(binding)?; + if digest(&bytes(&pins.creation)?) != binding.creation_digest + || digest(&bytes(¤t.creation)?) != binding.creation_digest + { + return Err(invalid("Checkout retirement creation bytes changed")); + } + Ok(()) +} +fn claim_name(plan: &CheckoutRetirementPlan) -> String { + format!("{}.lock", plan.binding.owner_id) +} +fn ack_name(plan: &CheckoutRetirementPlan) -> String { + format!("{}.snapshot-ack", plan.binding.owner_id) +} +fn absent(dir: &Dir, name: &str) -> io::Result<()> { + match dir.symlink_metadata(name) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + Ok(_) => Err(invalid("Retired checkout metadata route was replaced")), + } +} +fn verify_leaf( + leaf: Option<&Leaf>, + pins: &Pins, + name: &str, + expected: &CheckoutRetirementFile, +) -> io::Result<()> { + match leaf { + Some(leaf) => leaf.verify(pins, name, expected), + None => absent(&pins.directory, name), + } +} + +#[cfg(test)] +mod tests; + +/// Retains native evidence through a trusted adapter's last filesystem call. +/// It supplies no Store receipt, reference, retention or source-mutation authority. +pub struct CheckoutRetirementGuard { + claims: Vec, + repositories: BTreeMap, +} +impl CheckoutRetirementGuard { + /// Every claim must already carry its exact consuming acknowledgement. The + /// caller additionally retains the original snapshot lifetime and Store proof. + pub fn prepare(claims: Vec) -> io::Result { + if claims.len() > LIMIT { + return Err(invalid("Checkout retirement resource limit exceeded")); + } + let mut total = 0u64; + for claim in &claims { + let ack = claim.acknowledgement.as_ref().ok_or_else(|| { + invalid("Checkout retirement requires an existing acknowledgement") + })?; + for file in [&*claim.lease.file, ack.file_for_retirement()] { + total = total + .checked_add(file.metadata()?.len()) + .filter(|bytes| *bytes <= TOTAL_BYTES) + .ok_or_else(|| invalid("Checkout retirement total bytes exceed limit"))?; + } + } + let mut repositories = BTreeMap::::new(); + for claim in &claims { + let binding = claim.binding(); + if !repositories.contains_key(&binding.common_dir) { + repositories.insert( + binding.common_dir.clone(), + RepositoryPin::open(binding, None)?, + ); + } + } + let claims = claims + .into_iter() + .map(|claim| { + let repository = &repositories[&claim.binding().common_dir]; + Claim::capture(claim, repository.identity.clone()) + }) + .collect::>>()?; + let guard = Self { + claims, + repositories, + }; + guard.validate()?; + guard.verify()?; + Ok(guard) + } + /// Missing leaves are authorized progress ONLY by an exact already committed + /// Store metadata plan. The trusted adapter must attest that before calling. + pub fn resume_committed(plans: Vec) -> io::Result { + CheckoutRetirementPlan::validate_set(&plans)?; + let mut repositories = BTreeMap::::new(); + for plan in &plans { + match repositories.get(&plan.binding.common_dir) { + Some(repository) if repository.identity != plan.repository_lock_identity => { + return Err(invalid("Checkout retirement repository plans differ")); + } + Some(_) => {} + None => { + repositories.insert( + plan.binding.common_dir.clone(), + RepositoryPin::open(&plan.binding, Some(&plan.repository_lock_identity))?, + ); + } + } + } + let guard = Self { + claims: plans + .into_iter() + .map(Claim::resume) + .collect::>()?, + repositories, + }; + guard.verify()?; + Ok(guard) + } + pub fn plans(&self) -> Vec { + self.claims.iter().map(|claim| claim.plan.clone()).collect() + } + fn validate(&self) -> io::Result<()> { + CheckoutRetirementPlan::validate_set(&self.plans()) + } + pub fn verify(&self) -> io::Result<()> { + for repository in self.repositories.values() { + repository.verify()?; + } + for claim in &self.claims { + claim.verify()?; + } + Ok(()) + } + /// One exact unlink per call, acknowledgement before original claim. Callers + /// re-attest their retained Store/native/retention scope before EVERY call. + pub fn remove_next(&mut self) -> io::Result { + self.verify()?; + for acknowledgement in [true, false] { + for claim in &mut self.claims { + let (leaf, name) = if acknowledgement { + (&mut claim.acknowledgement, ack_name(&claim.plan)) + } else { + (&mut claim.claim, claim_name(&claim.plan)) + }; + if let Some(leaf) = leaf.as_mut().filter(|leaf| leaf.present) { + claim.pins.directory.remove_file(name)?; + super::super::publication::sync_directory(&claim.pins.directory)?; + leaf.present = false; + self.verify()?; + return Ok(true); + } + } + } + // Missing planned names may have been observed after an interrupted + // unlink whose first directory sync failed. Bind their absence now. + for claim in &self.claims { + super::super::publication::sync_directory(&claim.pins.directory)?; + } + self.verify()?; + Ok(false) + } +} diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/retirement/native/tests.rs b/crates/cyber-core/src/worktrees/activity/snapshot/retirement/native/tests.rs new file mode 100644 index 000000000..19204b9b6 --- /dev/null +++ b/crates/cyber-core/src/worktrees/activity/snapshot/retirement/native/tests.rs @@ -0,0 +1,310 @@ +//! Native primitive tests only; actual Store/receipt authorization lives in App tests. +use super::*; +use crate::worktrees::{CheckoutAcknowledgement, CheckoutOwner, Managed, RepositoryLock}; + +struct Fixture { + temp: tempfile::TempDir, + managed: Managed, +} +impl Drop for Fixture { + fn drop(&mut self) { + if std::thread::panicking() { + self.temp.disable_cleanup(true); + eprintln!( + "Preserved checkout retirement fixture at {}", + self.temp.path().display() + ); + } + } +} +impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let common_dir = root.join("common"); + let path = root.join("work"); + std::fs::create_dir_all(common_dir.join("cyber-worktrees")).unwrap(); + std::fs::create_dir(&path).unwrap(); + let managed = Managed { + id: "wt_fixture".into(), + name: "fixture".into(), + path, + branch: "cyber/fixture".into(), + base: "main".into(), + common_dir, + ready: true, + included: vec![], + }; + std::fs::write( + managed.common_dir.join("cyber-worktrees/fixture.json"), + serde_json::to_vec(&managed).unwrap(), + ) + .unwrap(); + std::fs::write(managed.common_dir.join("cyber-worktree.lock"), "").unwrap(); + Self { temp, managed } + } + fn claim(&self, activity: &str) -> SnapshotClaim { + let owner = CheckoutOwner { + activity_id: activity.into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: self.managed.path.clone(), + lifetime_digest: "a".repeat(64), + }; + super::super::super::publication::publish(&self.managed, activity, Some(owner)) + .unwrap() + .retain_snapshot() + .unwrap() + } + fn acknowledged(&self, activity: &str) -> SnapshotClaim { + let mut claim = self.claim(activity); + CheckoutAcknowledgement::new(claim.binding(), &"a".repeat(64), &"b".repeat(64)) + .unwrap() + .publish(&mut claim) + .unwrap(); + claim + } + fn directory(&self) -> PathBuf { + self.managed + .common_dir + .join("cyber-worktree-activity/wt_fixture") + } + fn plans(&self) -> Vec { + CheckoutRetirementGuard::prepare(vec![self.acknowledged("op_first")]) + .unwrap() + .plans() + } +} +fn remove_all(guard: &mut CheckoutRetirementGuard) -> usize { + let mut count = 0; + while guard.remove_next().unwrap() { + count += 1; + } + count +} + +#[test] +fn exact_retirement_preserves_shared_creation_unknown_activity_and_source() { + let f = Fixture::new(); + let mut guard = CheckoutRetirementGuard::prepare(vec![f.acknowledged("op_first")]).unwrap(); + let creation = f.managed.common_dir.join("cyber-worktrees/fixture.json"); + let bytes = std::fs::read(&creation).unwrap(); + let unknown = f.directory().join("op_unknown.lock"); + std::fs::write(&unknown, b"independent unknown evidence").unwrap(); + std::fs::write(f.managed.path.join("owner"), b"later owner source").unwrap(); + assert!( + RepositoryLock::try_acquire(&f.managed.common_dir) + .unwrap() + .is_none() + ); + assert_eq!(remove_all(&mut guard), 2); + guard.verify().unwrap(); + assert!( + RepositoryLock::try_acquire(&f.managed.common_dir) + .unwrap() + .is_none() + ); + assert_eq!(std::fs::read(&creation).unwrap(), bytes); + assert_eq!( + std::fs::read(&unknown).unwrap(), + b"independent unknown evidence" + ); + assert_eq!( + std::fs::read(f.managed.path.join("owner")).unwrap(), + b"later owner source" + ); + drop(guard); + assert!( + RepositoryLock::try_acquire(&f.managed.common_dir) + .unwrap() + .is_some() + ); +} + +#[test] +fn frozen_native_frame_resumes_each_leaf_cut_and_repeated_completion_without_recreation() { + for cut in 0..=2 { + let f = Fixture::new(); + let mut guard = CheckoutRetirementGuard::prepare(vec![f.acknowledged("op_first")]).unwrap(); + let frozen = serde_json::to_vec(&guard.plans()).unwrap(); + for _ in 0..cut { + assert!(guard.remove_next().unwrap()); + } + drop(guard); + let plans = serde_json::from_slice(&frozen).unwrap(); + let mut reopened = CheckoutRetirementGuard::resume_committed(plans).unwrap(); + assert_eq!(remove_all(&mut reopened), 2 - cut); + assert_eq!(serde_json::to_vec(&reopened.plans()).unwrap(), frozen); + drop(reopened); + let mut repeated = + CheckoutRetirementGuard::resume_committed(serde_json::from_slice(&frozen).unwrap()) + .unwrap(); + assert_eq!(remove_all(&mut repeated), 0); + } +} + +#[test] +fn two_consumed_owners_share_the_repository_lock_and_keep_their_inventory_order() { + let f = Fixture::new(); + let mut guard = CheckoutRetirementGuard::prepare(vec![ + f.acknowledged("op_first"), + f.acknowledged("op_second"), + ]) + .unwrap(); + let plans = guard.plans(); + assert_eq!(plans[0].binding.owner_id, "op_first"); + assert_eq!(plans[1].binding.owner_id, "op_second"); + assert_eq!( + plans[0].repository_lock_identity, + plans[1].repository_lock_identity + ); + assert!(guard.remove_next().unwrap() && guard.remove_next().unwrap()); + assert!( + f.directory().join("op_first.lock").exists() + && f.directory().join("op_second.lock").exists() + ); + assert!( + !f.directory().join("op_first.snapshot-ack").exists() + && !f.directory().join("op_second.snapshot-ack").exists() + ); + assert_eq!(remove_all(&mut guard), 2); +} + +#[test] +fn missing_acknowledgement_busy_repository_and_live_original_refuse_without_unlink() { + let f = Fixture::new(); + assert!(CheckoutRetirementGuard::prepare(vec![f.claim("op_unacknowledged")]).is_err()); + let claim = f.acknowledged("op_first"); + let lock = RepositoryLock::try_acquire(&f.managed.common_dir) + .unwrap() + .unwrap(); + assert!(CheckoutRetirementGuard::prepare(vec![claim]).is_err()); + drop(lock); + let mut proof = super::super::super::reacquire( + &f.managed, + &{ + let raw = std::fs::read(f.directory().join("op_first.lock")).unwrap(); + serde_json::from_slice::(&raw) + .unwrap() + .snapshot_binding + .unwrap() + }, + &CheckoutOwner { + activity_id: "op_first".into(), + session_id: "ses_fixture".into(), + session_created_id: "evt_fixture".into(), + directory: f.managed.path.clone(), + lifetime_digest: "a".repeat(64), + }, + ) + .unwrap(); + CheckoutAcknowledgement::new(proof.binding(), &"a".repeat(64), &"b".repeat(64)) + .unwrap() + .require_existing(&mut proof) + .unwrap(); + let plans = CheckoutRetirementGuard::prepare(vec![proof]) + .unwrap() + .plans(); + let live = super::super::super::reacquire( + &f.managed, + &plans[0].binding, + plans[0].binding.lifetime.as_ref().unwrap(), + ) + .unwrap(); + assert!(CheckoutRetirementGuard::resume_committed(plans).is_err()); + assert!( + f.directory().join("op_first.lock").exists() + && f.directory().join("op_first.snapshot-ack").exists() + ); + drop(live); +} + +#[test] +fn replacement_changed_bytes_alias_and_same_owner_construction_are_preserved() { + for kind in ["replacement", "changed", "alias", "construction", "context"] { + let f = Fixture::new(); + let mut plans = f.plans(); + let ack = f.directory().join("op_first.snapshot-ack"); + let before = std::fs::read(&ack).unwrap(); + match kind { + "replacement" => { + std::fs::rename(&ack, f.temp.path().join("retained-ack")).unwrap(); + std::fs::write(&ack, &before).unwrap(); + } + "changed" => std::fs::write(&ack, b"changed native evidence").unwrap(), + "alias" => std::fs::hard_link(&ack, f.temp.path().join("alias")).unwrap(), + "construction" => std::fs::write( + f.directory() + .join(format!("ack_preparing_8_op_first_{}", ulid::Ulid::new())), + b"unknown constructor", + ) + .unwrap(), + "context" => plans[0].receipt_digest = "c".repeat(64), + _ => unreachable!(), + } + assert!( + CheckoutRetirementGuard::resume_committed(plans).is_err(), + "{kind}" + ); + assert!(ack.exists() && f.directory().join("op_first.lock").exists()); + } +} + +#[test] +fn fifo_acknowledgement_refuses_without_waiting_for_a_writer() { + use std::os::unix::fs::FileTypeExt; + let f = Fixture::new(); + let plans = f.plans(); + let ack = f.directory().join("op_first.snapshot-ack"); + std::fs::rename(&ack, f.temp.path().join("retained-ack")).unwrap(); + assert!( + std::process::Command::new("mkfifo") + .arg(&ack) + .status() + .unwrap() + .success() + ); + let (send, receive) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + send.send(CheckoutRetirementGuard::resume_committed(plans).is_err()) + .unwrap(); + }); + assert!( + receive + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("native metadata open blocked on a FIFO") + ); + worker.join().unwrap(); + assert!(ack.symlink_metadata().unwrap().file_type().is_fifo()); + assert!(f.directory().join("op_first.lock").exists()); +} + +#[test] +fn changed_original_context_refuses_before_freezing_or_removing_metadata() { + let f = Fixture::new(); + let claim = f.acknowledged("op_first"); + let path = f.directory().join("op_first.lock"); + let mut record: UseRecord = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + record.settled = true; + let changed = serde_json::to_vec(&record).unwrap(); + std::fs::write(&path, &changed).unwrap(); + assert!(CheckoutRetirementGuard::prepare(vec![claim]).is_err()); + assert_eq!(std::fs::read(path).unwrap(), changed); + assert!(f.directory().join("op_first.snapshot-ack").exists()); +} + +#[test] +fn native_plan_data_bounds_reject_oversized_leaves_and_resource_vectors_before_native_io() { + let f = Fixture::new(); + let plan = f.plans().remove(0); + let mut oversized = plan.clone(); + oversized.claim.bytes = 1024 * 1024 + 1; + assert!(oversized.validate().is_err()); + let duplicate = vec![plan.clone(), plan.clone()]; + assert!(CheckoutRetirementPlan::validate_set(&duplicate).is_err()); + let oversized = vec![plan; 8193]; + let error = CheckoutRetirementPlan::validate_set(&oversized).unwrap_err(); + assert!(error.to_string().contains("resource limit")); + assert!(f.directory().join("op_first.lock").exists()); + assert!(f.directory().join("op_first.snapshot-ack").exists()); +} diff --git a/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs b/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs index 9aa14dcde..e4e537dfd 100644 --- a/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs +++ b/crates/cyber-server/src/runtime/snapshot_reclamation/metadata.rs @@ -9,15 +9,19 @@ pub const PLANNED: &str = "snapshot.reclamation.metadata.planned.1"; pub const COMPLETED: &str = "snapshot.reclamation.metadata.completed.1"; const FILE_LIMIT: usize = 10; const FILE_BYTES: u64 = 8 * 1024 * 1024; -const ENCODED_LIMIT: usize = 16 * 1024; +const ENCODED_LIMIT: usize = 32 * 1024 * 1024; +const LEGACY_ENCODED_LIMIT: usize = 16 * 1024; -struct EncodingLimit(usize); +struct EncodingLimit { + used: usize, + limit: usize, +} impl std::io::Write for EncodingLimit { fn write(&mut self, bytes: &[u8]) -> std::io::Result { - self.0 = self - .0 + self.used = self + .used .checked_add(bytes.len()) - .filter(|size| *size <= ENCODED_LIMIT) + .filter(|size| *size <= self.limit) .ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -30,8 +34,8 @@ impl std::io::Write for EncodingLimit { Ok(()) } } -fn bounded_encoding(value: &impl Serialize) -> Result<(), String> { - serde_json::to_writer(EncodingLimit(0), value) +fn bounded_encoding(value: &impl Serialize, limit: usize) -> Result<(), String> { + serde_json::to_writer(EncodingLimit { used: 0, limit }, value) .map_err(|_| "Metadata retirement encoding exceeds limit".into()) } @@ -62,10 +66,14 @@ pub struct Plan { pub version: u32, pub payload_plan_id: String, pub files: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub checkouts: Vec, } impl Plan { fn shape(&self) -> Result<(), StoreError> { - if self.version != 1 + if !matches!(self.version, 1 | 2) + || (self.version == 1) != self.checkouts.is_empty() + || self.checkouts.len() > 8192 || self.files.is_empty() || self.files.len() > FILE_LIMIT || !self @@ -96,8 +104,33 @@ impl Plan { )); } } + cyber_core::worktrees::CheckoutRetirementPlan::validate_set(&self.checkouts) + .map_err(|error| invalid(error.to_string()))?; + for plan in &self.checkouts { + for file in [&plan.claim, &plan.acknowledgement] { + if self.files.iter().any(|local| { + local.identity + == file + .identity + .split(':') + .take(4) + .collect::>() + .join(":") + }) { + return Err(invalid("Aliased local and checkout retirement metadata")); + } + } + } // Bound all strings before allocating their serialized representation. - bounded_encoding(self).map_err(invalid)?; + bounded_encoding( + self, + if self.version == 1 { + LEGACY_ENCODED_LIMIT + } else { + ENCODED_LIMIT + }, + ) + .map_err(invalid)?; self.total_bytes().map(drop) } pub fn validate(&self, payload: &PayloadPlan) -> Result<(), StoreError> { @@ -108,16 +141,33 @@ impl Plan { } let owners = payload.receipt.owners()?; for owner in &owners { + let plans = self + .checkouts + .iter() + .filter(|plan| plan.binding.owner_id == owner.origin.activity_id) + .collect::>(); + let bindings = plans + .iter() + .map(|plan| plan.binding.clone()) + .collect::>(); if owner .lifetime - .checkout_resources_digest(&[]) + .checkout_resources_digest(&bindings) .map_err(invalid)? != owner.resources_digest { return Err(invalid( - "Metadata retirement needs an empty complete checkout inventory", + "Metadata retirement checkout inventory differs from consumption", )); } + let receipt_digest = checkout_receipt_digest(&payload.receipt).map_err(invalid)?; + for plan in plans { + if plan.resources_digest != owner.resources_digest + || plan.receipt_digest != receipt_digest + { + return Err(invalid("Metadata retirement checkout disposition differs")); + } + } for role in [Role::Intent, Role::Lifetime, Role::SourcePhase] { if !self .files @@ -128,6 +178,13 @@ impl Plan { } } } + if self.checkouts.iter().any(|plan| { + !owners + .iter() + .any(|owner| owner.origin.activity_id == plan.binding.owner_id) + }) { + return Err(invalid("Metadata retirement has a foreign checkout owner")); + } for file in &self.files { let owner = owners .iter() @@ -153,9 +210,16 @@ impl Plan { Ok(()) } pub fn total_bytes(&self) -> Result { - self.files.iter().try_fold(0u64, |sum, file| { + let local = self.files.iter().try_fold(0u64, |sum, file| { sum.checked_add(file.bytes) .ok_or_else(|| invalid("Metadata retirement size overflow")) + })?; + self.checkouts.iter().try_fold(local, |sum, plan| { + sum.checked_add( + plan.total_bytes() + .map_err(|error| invalid(error.to_string()))?, + ) + .ok_or_else(|| invalid("Metadata retirement size overflow")) }) } /// Re-attest at the writer around a separately verified native callback. @@ -188,6 +252,14 @@ impl Plan { payload.authorize(db) } } +fn checkout_receipt_digest(receipt: &Receipt) -> Result { + match receipt { + Receipt::Restore(receipt) => receipt.checkout_acknowledgement_digest(), + Receipt::Preparation(receipt) => receipt.checkout_acknowledgement_digest(), + Receipt::Recovery(receipt) => receipt.checkout_acknowledgement_digest(), + } +} + fn native_identity(value: &str) -> bool { if value.len() > 128 { return false; @@ -281,7 +353,19 @@ pub(super) fn register(registry: &mut EventRegistry) { { return Err("Invalid metadata retirement file bounds".into()); } - bounded_encoding(value)?; + if value.get("checkouts").is_some_and(|checkouts| { + checkouts.as_array().is_none_or(|plans| plans.len() > 8192) + }) { + return Err("Invalid checkout retirement resource bounds".into()); + } + bounded_encoding( + value, + if value.get("version").and_then(serde_json::Value::as_u64) == Some(2) { + ENCODED_LIMIT + } else { + LEGACY_ENCODED_LIMIT + }, + )?; serde_json::from_value::(value.clone()) .map_err(|e| e.to_string())? .shape() diff --git a/crates/cyber-snapshot/src/copy/reclamation/checkouts.rs b/crates/cyber-snapshot/src/copy/reclamation/checkouts.rs index c8670fea0..46b4d88cd 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/checkouts.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/checkouts.rs @@ -10,6 +10,9 @@ pub(super) struct Checkouts { } impl Checkouts { + pub fn into_claims(self) -> Vec { + self.claims + } pub fn acquire( store: &Store, lifetime: &super::super::lifetime::Lifetime, diff --git a/crates/cyber-snapshot/src/copy/reclamation/retirement.rs b/crates/cyber-snapshot/src/copy/reclamation/retirement.rs index 61fc24127..bd9f976ab 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/retirement.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/retirement.rs @@ -29,16 +29,6 @@ pub(crate) fn retire( if !due(config.as_ref(), &store.root, settlement.settled_at_ms) { return Ok(0); } - for owner in receipt.owners().map_err(storage::fail)? { - if owner.lifetime.owner.worktree_id.is_some() - || owner.lifetime.checkout_resources_digest(&[])? != owner.resources_digest - { - return Err("Managed checkout metadata retirement is not yet available".into()); - } - } - if record.metadata.is_none() { - verify_initial(&store, &record)?; - } let prepared = native::Prepared::new(store, workspace, config, settlement.settled_at_ms, record)?; let plan = prepared.plan.clone(); @@ -46,8 +36,8 @@ pub(crate) fn retire( policy.retire(payload, plan, Box::new(move || prepared.delete())) } -fn verify_initial(store: &Store, retained: &records::Record) -> Result<(), String> { - let receipt = &retained.plan.receipt; +fn verify_initial(store: &Store, retained: &records::Plan) -> Result<(), String> { + let receipt = &retained.receipt; for owner in receipt.owners().map_err(storage::fail)? { store.verify_worker_namespace(owner.origin)?; let (_, journal) = store.journal(&owner.origin.binding.journal_id)?; @@ -69,17 +59,16 @@ fn verify_initial(store: &Store, retained: &records::Record) -> Result<(), Strin } fn initial_journal( store: &Store, - retained: &records::Record, + retained: &records::Plan, journal: &Journal, origin: &cyber_server::runtime::SnapshotOrigin, ) -> Result<(String, bool), String> { - match &retained.plan.receipt { + match &retained.receipt { Receipt::Restore(receipt) if ordinary(journal) => { Ok((receipt.intent_digest.clone(), !journal.changes.is_empty())) } Receipt::Preparation(receipt) if cancelled::verify(store, receipt, journal)? => Ok(( retained - .plan .intent_digest .clone() .ok_or("Missing cancellation intent digest")?, diff --git a/crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs b/crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs index ecb294ccf..05c593f7e 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/retirement/native.rs @@ -1,5 +1,6 @@ //! Native guards and fixed semantic routes; no source image is opened or installed. use super::*; +use cyber_core::worktrees::CheckoutRetirementGuard; use cyber_server::runtime::{SnapshotLifetimeBinding, SnapshotOrigin}; use metadata::{File as Entry, Plan as MetadataPlan, Role}; use std::os::unix::fs::FileExt; @@ -7,6 +8,7 @@ use std::os::unix::fs::FileExt; struct Owner { origin: SnapshotOrigin, binding: SnapshotLifetimeBinding, + resources_digest: String, journal: Option, journal_generation: Option, _lifetime: Option, @@ -24,6 +26,7 @@ pub(super) struct Prepared { workers: Dir, workers_generation: String, owners: Vec, + checkouts: CheckoutRetirementGuard, leaves: Vec, _workspace: tokio::sync::OwnedMutexGuard<()>, config: Arc, @@ -57,19 +60,43 @@ impl Prepared { owners.push(Owner { origin: evidence.origin.clone(), binding: evidence.lifetime.clone(), + resources_digest: evidence.resources_digest.to_owned(), journal, journal_generation, _lifetime: lifetime, }); } + let checkouts = match &frozen { + Some(plan) => CheckoutRetirementGuard::resume_committed(plan.checkouts.clone()) + .map_err(storage::fail)?, + None => { + let mut claims = Vec::new(); + for owner in &owners { + let lifetime = owner + ._lifetime + .as_ref() + .ok_or("Missing original metadata lifetime")?; + let proof = super::super::checkouts::Checkouts::acquire( + &store, + lifetime, + &owner.binding, + &owner.resources_digest, + &super::super::checkouts::receipt_digest(&record.plan.receipt)?, + )?; + claims.extend(proof.into_claims()); + } + CheckoutRetirementGuard::prepare(claims).map_err(storage::fail)? + } + }; let entries = match &frozen { Some(plan) => plan.files.clone(), None => capture(&store, &workers, &owners, &record.plan.receipt)?, }; let plan = MetadataPlan { - version: 1, + version: if checkouts.plans().is_empty() { 1 } else { 2 }, payload_plan_id: records::Plan::id(&record.plan.receipt).map_err(storage::fail)?, files: entries, + checkouts: checkouts.plans(), }; plan.validate(&record.plan).map_err(storage::fail)?; if frozen.as_ref().is_some_and(|frozen| *frozen != plan) { @@ -91,12 +118,18 @@ impl Prepared { pin, }); } + // Freeze native identities/generations first, then re-attest original + // typed intent/phase context. verify() rejects any subsequent mutation. + if frozen.is_none() { + super::verify_initial(&store, &record.plan)?; + } leaves.sort_by_key(|leaf| priority(leaf.entry.role)); let prepared = Self { workers_generation: directory_generation(&workers)?, store, workers, owners, + checkouts, leaves, plan, payload: record.plan, @@ -112,12 +145,7 @@ impl Prepared { if !due(self.config.as_ref(), &self.store.root, self.settled) { return Err("Snapshot retention no longer permits metadata retirement".into()); } - if !cyber_core::worktrees::Repository::managed_locations_at(&self.store.root) - .map_err(storage::fail)? - .is_empty() - { - return Err("Metadata retirement checkout registration changed".into()); - } + self.verify_checkouts()?; let fresh = Store::existing( &self.store.data_path, &self.store.root.to_string_lossy(), @@ -142,6 +170,28 @@ impl Prepared { } self.inventory() } + fn verify_checkouts(&self) -> Result<(), String> { + let current = cyber_core::worktrees::Repository::managed_locations_at(&self.store.root) + .map_err(storage::fail)?; + for owner in &self.owners { + let plans = self + .plan + .checkouts + .iter() + .filter(|plan| plan.binding.owner_id == owner.origin.activity_id) + .collect::>(); + if current.len() != plans.len() + || !current.iter().zip(plans).all(|((_, managed), plan)| { + managed.common_dir == plan.binding.common_dir + && managed.id == plan.binding.worktree_id + && managed.name == plan.binding.name + }) + { + return Err("Metadata retirement checkout registration changed".into()); + } + } + self.checkouts.verify().map_err(storage::fail) + } fn verify_owner( &self, owner: &Owner, @@ -208,6 +258,12 @@ impl Prepared { } pub fn delete(mut self) -> Result { self.verify()?; + loop { + self.verify()?; + if !self.checkouts.remove_next().map_err(storage::fail)? { + break; + } + } for index in 0..self.leaves.len() { self.verify()?; let leaf = &self.leaves[index]; @@ -237,6 +293,10 @@ impl Prepared { self.owners[index].journal_generation = None; } } + // Resumption can observe already missing names before any unlink here; + // synchronize their exact parent directories before durable completion. + storage::sync(&self.workers)?; + storage::sync(&self.store.journals)?; self.verify()?; self.plan.total_bytes().map_err(storage::fail) } diff --git a/crates/cyber-store/migrations/20261110010000_snapshot_checkout_retirement.sql b/crates/cyber-store/migrations/20261110010000_snapshot_checkout_retirement.sql new file mode 100644 index 000000000..3b70a7017 --- /dev/null +++ b/crates/cyber-store/migrations/20261110010000_snapshot_checkout_retirement.sql @@ -0,0 +1,3 @@ +-- Version-2 receipt-bound metadata plans include complete checkout bindings and +-- native claim/acknowledgement observations. Older catalogs must remain read-only. +SELECT 1; diff --git a/crates/cyber-store/src/migrate.rs b/crates/cyber-store/src/migrate.rs index 2b12201bd..c7ac8990a 100644 --- a/crates/cyber-store/src/migrate.rs +++ b/crates/cyber-store/src/migrate.rs @@ -209,6 +209,10 @@ pub(crate) const MIGRATIONS: &[Migration] = &[ id: "20261110000000_snapshot_metadata_retirement", sql: include_str!("../migrations/20261110000000_snapshot_metadata_retirement.sql"), }, + Migration { + id: "20261110010000_snapshot_checkout_retirement", + sql: include_str!("../migrations/20261110010000_snapshot_checkout_retirement.sql"), + }, ]; pub(crate) enum Outcome { @@ -655,13 +659,21 @@ mod tests { #[test] fn metadata_retirement_catalog_survives_reopen_and_preserves_unknown_evidence() { + for id in [ + "20261110000000_snapshot_metadata_retirement", + "20261110010000_snapshot_checkout_retirement", + ] { + metadata_catalog_fence(id); + } + } + fn metadata_catalog_fence(id: &str) { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("retirement.db"); let marker = temp.path().join("retained-intent.next"); std::fs::write(&marker, b"unknown native metadata").unwrap(); let at = MIGRATIONS .iter() - .position(|migration| migration.id == "20261110000000_snapshot_metadata_retirement") + .position(|migration| migration.id == id) .unwrap(); let mut db = Connection::open(&path).unwrap(); migrate(&mut db, &MIGRATIONS[..at]).unwrap(); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index ea41deaeb..888b42ed3 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -117,12 +117,12 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. files for empty checkout inventories. Periodic Store discovery resumes after active-intent deletion, including a tested real process kill; cap pressure can reclaim metadata before one capture retry. Completed acknowledgement returns - the retained Store receipt without reopening deleted files. Managed claims, - partial construction and complete receipt-kind crash/platform acceptance remain + the retained Store receipt without reopening deleted files. Remaining managed + claim crash cases, partial construction and full platform acceptance remain open. Real prepared-cancellation and paired keep/rollback CLI fixtures now exercise metadata retirement and receipt acknowledgement after restart, including no-follow checks for removed lifetime/phase files. These positive cases do not - establish the complete crash matrix or managed claim retirement. + establish the complete crash matrix. A separate real App regression reproduced 57,344 bytes of payload deletion when configuration resolved to null. Cleanup now preserves bytes initially and @@ -132,6 +132,21 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. retry. The earlier 14 migration tests cover the writer catalog. Fresh platform and broader current-source verification remain required before M6 acceptance. +9. **Managed retirement now freezes complete checkout bindings before deletion.** + A version-2 metadata plan binds each original claim and its existing consuming + acknowledgement, native generations/hashes and persistent repository-lock + identity. The native callback retains the original lifetime and all repository/ + leaf locks, removes acknowledgement before claim, preserves shared creation and + unrelated activity records, and synchronizes observed absence on resumption. + The Store writer re-attests receipt, retention and references; a new catalog + marker fences older writers. Version-1 plans retain their original meaning. + Real App/CLI managed ordinary restore, prepared cancellation and paired + keep/rollback positives now remove those leaves and acknowledge from the + retained receipt after restart without source replay. Guard reopen/cut, + alias/context/live-owner/construction and bounded FIFO cases pass locally. + Actual managed metadata-deletion process cuts, zero-payload/partial constructors, + old/deleted Locations and source-bound platforms still gate M6 acceptance. + Native run [42733865471](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42733865471) passes Linux x64/ARM and macOS ARM/Intel at `e2b3ba18`, including the earlier managed Git cleanup and structured CLI cases. It predates the owned intent-slot diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 95383dc2f..2b75ce39c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -1019,6 +1019,33 @@ callback. The first writer frame supports empty complete checkout inventories; managed checkout claim retirement, partial constructors and all actual native retirement/reconciliation/crash paths remain required before M6 acceptance. +#### Consumed managed checkout retirement + +The managed extension SHALL freeze the complete ordered checkout bindings for each +consumed owner with the metadata plan before the first unlink. Each binding SHALL +match its owner's consumed resource digest and lifetime, and its native claim and +existing acknowledgement SHALL match the same receipt digest. The plan SHALL bind +both leaves' native identity, generation, BLAKE3 content hash and bounded length, +plus the persistent repository-lock identity. Shape validation is not deletion +authority. Older Store writers SHALL refuse the extended protocol through the +migration catalog; version-1 empty-inventory plans retain their exact meaning. +The version-2 envelope SHALL bound the combined inventory to 8,192 checkout +resources, one MiB per native leaf, 64 MiB of checkout metadata in total and +32 MiB of encoded plan data. Legacy version-1 plans retain the 16 KiB envelope +bound. Overflow refuses before deletion. Historical path validation SHALL remain +data-only and portable; native acquisition additionally requires host-native routes. + +The native adapter SHALL retain all original lifetime locks, repository locks, +creation/namespace pins and surviving claim/acknowledgement locks through the last +filesystem call. It SHALL derive fixed semantic names from verified bindings, +revalidate registrations and every surviving leaf before each unlink, remove the +acknowledgement before its original claim and synchronize the activity directory. +Only an already committed exact plan permits missing leaves on restart. A changed +creation, route, byte sequence, generation, alias or live owner refuses. Unknown +construction entries for the same owner remain fenced; unrelated activity and +shared creation records remain intact. The callback remains serialized against +reference writers, with fresh retention and complete native-owner re-attestation. + ### Paired recovery payload cleanup Acknowledged keep/rollback cleanup SHALL bind the exact durable two-owner recovery diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index ba3ee1590..4386b6fe6 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1192,3 +1192,94 @@ HTTP/generated contract changes. Full workspace, SDK runtime, other CLI/TUI case all receipt-kind crash cuts and new-source platform acceptance remain unperformed here and required by their parent tasks. No M6, issue, PR or delivery completion is claimed. The preserved original pre-main-integration stash remains intact. + +Managed retirement implementation continuation: extend the approved M6 protocol +with a frozen complete checkout inventory, exact native claim/acknowledgement +observations and persistent repository-lock identity. Retain all native locks +through the final metadata call; remove only verified acknowledged leaves under +the Store writer's durable plan. Resumption may observe missing exact leaves only +after that plan, and same-owner construction/foreign/aliased/live evidence refuses. +Shared creation and unrelated activity records remain retained. Implement the core +native guard, extended writer frame/catalog fence, actual snapshot adapter and +ordinary/preparation/paired managed positives plus refusal and process-cut cases. +This is ongoing M6 implementation, not acceptance or a new parent completion. + +Managed checkout metadata retirement (2026-11-10): core now exposes a native guard +separate from its serializable observations. It derives fixed claim/acknowledgement +names, retains unique no-follow file locks and persistent repository locks, checks +native routes/creation bytes and every surviving generation before each unlink, +removes acknowledgement before original claim and preserves unrelated activity and +shared creation records. Same-owner unknown constructors refuse. Missing leaves +are progress only for the trusted adapter's already committed exact plan; ordinary +acquisition/acknowledgement is never synthesized during cleanup. Metadata opens +are nonblocking before regular-file validation, including a bounded FIFO regression. + +The internal metadata event's version-2 frame retains complete ordered checkout +bindings with per-owner consumed resource/receipt digests and exact leaf metadata. +It bounds resources to 8,192, each native leaf to one MiB, aggregate checkout bytes +to 64 MiB and encoding to 32 MiB. Legacy version-1 frames stay empty-inventory and +16 KiB bounded. Shape/uniqueness and exact consumption are validated at the Store +writer; no data frame itself grants native effects. Migration marker +20261110010000_snapshot_checkout_retirement makes older catalogs read-only and +preserves existing evidence across physical reopen. Historical absolute-path +shape validation is portable and data-only; native acquisition separately requires +host-native absolute routes. + +The actual snapshot adapter now prepares those guards from retained lifetimes, +complete resource publication and existing exact consuming acknowledgements, +commits the immutable Store plan, performs guarded claim and private metadata +retirement under the writer, and resumes from frozen bindings after phase/intent +removal. Repository locks remain through its last filesystem call. Shared +namespace/record entries are synchronized before planning, and exact parent +directories are also synchronized when resumption merely observes missing names. +Review tightened capture ordering: native identities/hashes/generations are frozen +before typed original context is re-attested, then fresh pins reject later changes. +Changed intent/admission and original-claim context regressions preserve bytes and +refuse before planning. The default owned-Git admission gate remains off. + +Actual local positives cover managed ordinary restore in the default data layout, +managed prepared cancellation, and both keep/rollback after real partial source +replacement kills. They verify claim/acknowledgement disappearance, exact completion, +no-op repeats, protected index/config/creation bytes and receipt-only lookup/ +acknowledgement after restarting. Later source edits/links, transcript and inference +counts remain unchanged. Native guard tests cover both per-leaf cuts through reopen, +multiple consumed owners sharing one repository lock, live/busy exclusion, +replacement/changed/aliased/context/constructor refusal, FIFO nonblocking and data +bounds. These primitive reopen cuts are not managed Store/process-death acceptance. +The prior unrelated probe panic is caught/expected; two test-helper failures were +retained as failures (.tmpvlw4ci and .tmptdEG3s), then corrected to avoid recreating +retired journals or killing an already stopped service. App/core failed fixture +paths are now retained for native CI alongside the existing CLI evidence. + +Final local managed-retirement verification: 22 selected core worktree tests pass +(two existing subprocess fixtures ignored at top level), including nine new +retirement/portable-path cases; all 14 migration cases pass. Both real App Git +runtime cases and all 38 App reclamation cases pass. The final three managed CLI +cases pass in 292.72 seconds, including the caught fixture-preservation probe and +both keep/rollback source-kill actions. A later sample attempt finds the CLI process +already terminal; the verified filesystem wait is not treated as a timeout or +restarted. The snapshot feature package passes 154 library, 31 file-copy and 22 Git +cases (nine library/one Git subprocess fixtures intentionally ignored at top level; +Windows-only/doc binaries select zero here). All-target core/Store/server/snapshot/ +App/CLI Clippy passes with warnings denied, as do format/whitespace, SDK generation +consistency, strict OpenSpec (100 items), spec lint (0 errors/25 warnings), and Ruby +workflow YAML parsing/guard-step selection. macOS C sandbox deprecations and linker +unwind-size diagnostics are observed without assigning their origin. Build-only +CLI artifacts were cleaned with cargo clean after verifying no CLI test/service +was live; original source/evidence and the pre-main-integration stash are preserved. + +Code-review-skill summary for this checkpoint: architecture follows the existing +native core / runtime receipt / App Store-writer / snapshot adapter boundaries. +Native plans are data only; publication uses existing consumption markers and no +source replay or capability activation. Review identified and corrected capture +ordering and missing-parent-sync gaps; context/bounds/refusal tests pass. Critical: +0. Major: 1 required acceptance gap (managed metadata-deletion process/crash matrix +and current-source platforms). Minor/nits: 0. Verdict: REQUEST CHANGES before full +recovery delivery, while this scoped checkpoint is reviewable. Next: actual managed +metadata process cuts/retention-loss/reader cases, zero-payload and partial +construction, old/deleted Location maintenance and complete native acceptance. +Full workspace, SDK runtime and other CLI/TUI scenarios were not rerun here. +M6, parent tasks and issue #93 remain open; no PR, merge or delivery is claimed. + +The final OpenAPI artifact consistency test also passes (one exact HTTP case). +No generated HTTP/SDK schema changes are required by this internal protocol. From 1f5b704d1e89919410908e6835052c54b103d548 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 04:49:55 -0400 Subject: [PATCH 68/87] Verify managed metadata retirement after real process death --- .github/workflows/non-git-snapshot-proof.yml | 1 + .../cyber-app/tests/snapshot_git_runtime.rs | 134 ++++---- .../tests/support/managed_retirement_cut.rs | 311 ++++++++++++++++++ docs/goal-delivery.md | 7 +- .../complete-daily-coding-surfaces/tasks.md | 41 +++ 5 files changed, 431 insertions(+), 63 deletions(-) create mode 100644 crates/cyber-app/tests/support/managed_retirement_cut.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 8964deaba..16ea63161 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -7,6 +7,7 @@ on: - crates/cyber-app/src/** - crates/cyber-app/tests/support/scripted_model.rs - crates/cyber-app/tests/support/snapshot*.rs + - crates/cyber-app/tests/support/managed_retirement_cut.rs - crates/cyber-server/src/runtime/snapshot*.rs - crates/cyber-server/src/runtime/snapshot_reclamation/** - crates/cyber-server/src/runtime/rewind*.rs diff --git a/crates/cyber-app/tests/snapshot_git_runtime.rs b/crates/cyber-app/tests/snapshot_git_runtime.rs index 7f9ddc756..1e45da4ca 100644 --- a/crates/cyber-app/tests/snapshot_git_runtime.rs +++ b/crates/cyber-app/tests/snapshot_git_runtime.rs @@ -3,6 +3,8 @@ any(target_os = "linux", target_os = "macos") ))] //! The real App/Store/worker path in an explicitly non-default acceptance build. +#[path = "support/managed_retirement_cut.rs"] +mod managed_retirement_cut; #[path = "support/scripted_model.rs"] mod scripted_model; use cyber_app::{App, AppOptions}; @@ -93,6 +95,42 @@ impl Fixture { session, } } + async fn managed() -> Self { + let mut f = Self::new("sha1", false).await; + std::fs::write(f.project.join("tracked"), "initial").unwrap(); + git(&f.project, &["add", "tracked"]); + git( + &f.project, + &[ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "-c", + "commit.gpgsign=false", + "commit", + "--quiet", + "-m", + "Initial", + ], + ); + let created = f + .client + .post( + "/worktrees", + json!({"name":"managed", "session":{"model":"test/main"}}), + ) + .await + .unwrap(); + f.directory = PathBuf::from(created["data"]["worktree"]["path"].as_str().unwrap()); + f.session = created["data"]["session"]["id"] + .as_str() + .unwrap() + .to_owned(); + f.client = Client::embedded(f.app.embedded()).at(f.directory.to_str().unwrap()); + assert!(f.directory.starts_with(f.paths.data.join("worktrees"))); + f + } async fn prompt(&self, text: &str, count: usize) -> String { self.client .post( @@ -190,75 +228,19 @@ fn image(value: u8) -> Vec { #[tokio::test] async fn managed_worktree_in_default_data_directory_restores_through_owned_worker() { - let mut f = Fixture::new("sha1", false).await; - std::fs::write(f.project.join("tracked"), "initial").unwrap(); - git(&f.project, &["add", "tracked"]); - git( - &f.project, - &[ - "-c", - "user.name=Fixture", - "-c", - "user.email=fixture@example.test", - "-c", - "commit.gpgsign=false", - "commit", - "--quiet", - "-m", - "Initial", - ], - ); - let created = f - .client - .post( - "/worktrees", - json!({"name":"managed", "session":{"model":"test/main"}}), - ) - .await - .unwrap(); - f.directory = PathBuf::from(created["data"]["worktree"]["path"].as_str().unwrap()); - f.session = created["data"]["session"]["id"] - .as_str() - .unwrap() - .to_owned(); - f.client = Client::embedded(f.app.embedded()).at(f.directory.to_str().unwrap()); - assert!(f.directory.starts_with(f.paths.data.join("worktrees"))); + let f = Fixture::managed().await; let index_path = PathBuf::from(git( &f.directory, &["rev-parse", "--path-format=absolute", "--git-path", "index"], )); let index = std::fs::read(&index_path).unwrap(); let config = std::fs::read(f.project.join(".git/config")).unwrap(); - std::fs::write(f.directory.join("tracked"), "before").unwrap(); - let first = f.prompt("before", 1).await; - std::fs::write(f.directory.join("tracked"), "after").unwrap(); - f.prompt("after", 2).await; - let mut request = json!({"message_id":first,"target":"code","action":"commit","expected_directory":f.directory}); - let preview = f - .client - .post( - &format!("/sessions/{}/rewind/preview", f.session), - request.clone(), - ) - .await - .unwrap(); - request["confirmation"] = preview["data"]["confirmation"].clone(); - tokio::time::timeout( - Duration::from_secs(30), - f.client - .post(&format!("/sessions/{}/rewind/confirm", f.session), request), - ) - .await - .expect("managed confirmation timed out") - .unwrap(); - assert_eq!( - std::fs::read_to_string(f.directory.join("tracked")).unwrap(), - "before" - ); - let receipt = f.receipt(); + let receipt = restore_tracked(&f).await; assert_eq!( serde_json::to_value(&receipt.origin).unwrap()["worktree_id"], - created["data"]["worktree"]["id"] + cyber_core::worktrees::Repository::managed_locations_at(&f.directory).unwrap()[0] + .1 + .id ); let checkout = receipt.checkout.as_ref().unwrap(); let journal = f.journal(&receipt); @@ -304,6 +286,36 @@ async fn managed_worktree_in_default_data_directory_restores_through_owned_worke f.app.runtime.shutdown().await; } +async fn restore_tracked(f: &Fixture) -> SnapshotRestoreReceipt { + std::fs::write(f.directory.join("tracked"), "before").unwrap(); + let first = f.prompt("before", 1).await; + std::fs::write(f.directory.join("tracked"), "after").unwrap(); + f.prompt("after", 2).await; + let mut request = json!({"message_id":first,"target":"code","action":"commit","expected_directory":f.directory}); + let preview = f + .client + .post( + &format!("/sessions/{}/rewind/preview", f.session), + request.clone(), + ) + .await + .unwrap(); + request["confirmation"] = preview["data"]["confirmation"].clone(); + tokio::time::timeout( + Duration::from_secs(30), + f.client + .post(&format!("/sessions/{}/rewind/confirm", f.session), request), + ) + .await + .expect("managed confirmation timed out") + .unwrap(); + assert_eq!( + std::fs::read_to_string(f.directory.join("tracked")).unwrap(), + "before" + ); + f.receipt() +} + async fn reclaim_restore_payloads(f: &Fixture, receipt: &SnapshotRestoreReceipt) { use cyber_server::runtime::snapshot_reclamation::Receipt; let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); diff --git a/crates/cyber-app/tests/support/managed_retirement_cut.rs b/crates/cyber-app/tests/support/managed_retirement_cut.rs new file mode 100644 index 000000000..d33f40a58 --- /dev/null +++ b/crates/cyber-app/tests/support/managed_retirement_cut.rs @@ -0,0 +1,311 @@ +//! Actual SIGKILL during managed metadata deletion, using only the configuration +//! callback to observe a real unlink. No native source or deletion hook is replaced. +use super::*; +use cyber_server::runtime::snapshot_reclamation::Receipt; +use cyber_snapshot::{GitSnapshots, reclamation::Policy}; +use std::os::unix::{fs::PermissionsExt, process::ExitStatusExt}; + +struct Control(tempfile::TempDir); +impl Drop for Control { + fn drop(&mut self) { + if std::thread::panicking() { + self.0.disable_cleanup(true); + eprintln!( + "Preserved managed retirement cut at {}", + self.0.path().display() + ); + } + } +} +fn maintenance(paths: &Paths, policy: Arc) -> Arc { + GitSnapshots::with_reclamation( + paths.data.clone(), + Arc::new(|_| json!({})), + vec![ + paths.config.clone(), + paths.state.clone(), + paths.cache.clone(), + paths.tmp.clone(), + ], + None, + Some(policy), + ) +} +fn sequence(store: &cyber_store::Store, session: &str) -> i64 { + let session = session.to_owned(); + store + .read(move |db| { + Ok(db.query_row( + "SELECT coalesce(max(seq),-1) FROM event WHERE aggregate_id=?1", + [session], + |row| row.get(0), + )?) + }) + .unwrap() +} + +#[tokio::test] +async fn managed_metadata_retirement_child() { + let Ok(control) = std::env::var("CYBER_MANAGED_METADATA_CUT_CONTROL") else { + return; + }; + let control = PathBuf::from(control); + let cut = std::env::var("CYBER_MANAGED_METADATA_CUT").unwrap(); + let mut f = Fixture::managed().await; + // The parent owns this fixture through killing/reopening, including any + // error before the observed cut. Keep all evidence on a child failure. + f._temp.disable_cleanup(true); + let receipt = restore_tracked(&f).await; + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + let snapshots = maintenance(&f.paths, policy.clone()); + let activity = receipt.origin.activity_id.clone(); + f.app + .store + .transaction(move |db| { + db.execute( + "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", + rusqlite::params![8i64 * 86400000, activity], + )?; + Ok(()) + }) + .unwrap(); + assert!( + snapshots + .reclaim(f.directory.to_str().unwrap()) + .await + .unwrap() + > 0 + ); + let journal = f.journal(&receipt); + let phase = journal + .parent() + .unwrap() + .parent() + .unwrap() + .join("workers") + .join( + receipt + .checkout + .as_ref() + .unwrap() + .lifetime + .source_phase + .as_ref() + .unwrap(), + ); + let phase: Value = serde_json::from_slice(&std::fs::read(phase).unwrap()).unwrap(); + let binding: cyber_core::worktrees::CheckoutBinding = + serde_json::from_value(phase["checkout_bindings"][0].clone()).unwrap(); + let activity = binding + .common_dir + .join("cyber-worktree-activity") + .join(&binding.worktree_id); + let watch = activity.join(format!( + "{}{}", + binding.owner_id, + if cut == "acknowledgement" { + ".snapshot-ack" + } else { + ".lock" + } + )); + write_fixture(&control, &f, &receipt, &journal, &binding); + let marker = control.join("ready"); + let config = Arc::new(move |_: &Path| { + if !watch.exists() { + std::fs::write(&marker, b"observed exact native checkout unlink").unwrap(); + loop { + std::thread::sleep(Duration::from_secs(1)); + } + } + json!({}) + }); + let snapshots = GitSnapshots::with_reclamation( + f.paths.data.clone(), + config, + vec![ + f.paths.config.clone(), + f.paths.state.clone(), + f.paths.cache.clone(), + f.paths.tmp.clone(), + ], + None, + Some(policy), + ); + snapshots + .retire(f.directory.to_str().unwrap(), Receipt::Restore(receipt)) + .await + .unwrap(); + panic!("managed metadata worker passed its selected cut"); +} +fn write_fixture( + control: &Path, + f: &Fixture, + receipt: &SnapshotRestoreReceipt, + journal: &Path, + binding: &cyber_core::worktrees::CheckoutBinding, +) { + let index = PathBuf::from(git( + &f.directory, + &["rev-parse", "--path-format=absolute", "--git-path", "index"], + )); + let config = f.project.join(".git/config"); + let creation = binding + .common_dir + .join("cyber-worktrees") + .join(format!("{}.json", binding.name)); + let protected = [index, config, creation] + .into_iter() + .map(|path| { + let bytes = std::fs::read(&path).unwrap(); + json!({"path":path,"bytes":bytes}) + }) + .collect::>(); + let data = json!({"root":f._temp.path(),"directory":f.directory,"journal":journal,"receipt":receipt, + "session":f.session,"session_sequence":sequence(&f.app.store,&f.session),"protected":protected}); + std::fs::write( + control.join("fixture.json"), + serde_json::to_vec(&data).unwrap(), + ) + .unwrap(); +} + +#[tokio::test] +async fn managed_metadata_process_death_resumes_frozen_checkouts_through_periodic_maintenance() { + for cut in ["acknowledgement", "claim"] { + let control = Control(tempfile::tempdir().unwrap()); + kill_at_cut(control.0.path(), cut).await; + resume(control.0.path(), cut).await; + } +} +async fn kill_at_cut(control: &Path, cut: &str) { + let log = std::fs::File::create(control.join("worker.log")).unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "managed_retirement_cut::managed_metadata_retirement_child", + "--nocapture", + ]) + .env("CYBER_MANAGED_METADATA_CUT_CONTROL", control) + .env("CYBER_MANAGED_METADATA_CUT", cut) + .env("TMPDIR", control) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(90); + let ready = loop { + if control.join("ready").exists() { + break true; + } + if child.try_wait().unwrap().is_some() || std::time::Instant::now() >= deadline { + break false; + } + tokio::time::sleep(Duration::from_millis(20)).await; + }; + let _ = child.kill(); + let status = child.wait().unwrap(); + assert!( + ready, + "missed {cut} native cut; retained {}: {}", + control.display(), + std::fs::read_to_string(control.join("worker.log")).unwrap() + ); + assert_eq!(status.signal(), Some(9)); +} +async fn resume(control: &Path, cut: &str) { + let data: Value = + serde_json::from_slice(&std::fs::read(control.join("fixture.json")).unwrap()).unwrap(); + let root = PathBuf::from(data["root"].as_str().unwrap()); + let directory = PathBuf::from(data["directory"].as_str().unwrap()); + let journal = PathBuf::from(data["journal"].as_str().unwrap()); + let receipt: SnapshotRestoreReceipt = serde_json::from_value(data["receipt"].clone()).unwrap(); + let paths = Paths { + data: root.join("data"), + config: root.join("config"), + state: root.join("state"), + cache: root.join("cache"), + tmp: root.join("tmp"), + }; + let store = Arc::new( + cyber_store::Store::open(cyber_store::StoreOptions::new( + DatabaseLocation::File(paths.data.join("cyber.db")), + cyber_server::runtime::Runtime::registry(), + )) + .unwrap(), + ); + let policy = cyber_app::snapshot_reclamation::policy(store.clone()); + let retained = Receipt::Restore(receipt.clone()); + let before = policy.record(&retained).unwrap().unwrap(); + assert_cut_state(&before, &journal, cut); + let frozen = before.metadata.unwrap(); + let binding = frozen.checkouts[0].binding.clone(); + let owner = root.join("later-owner"); + std::fs::write(&owner, b"owner after managed metadata process death").unwrap(); + std::fs::remove_file(directory.join("tracked")).unwrap(); + std::os::unix::fs::symlink(&owner, directory.join("tracked")).unwrap(); + std::fs::set_permissions(&owner, std::fs::Permissions::from_mode(0o0)).unwrap(); + let snapshots = maintenance(&paths, policy.clone()); + snapshots.gc().await; + let completed = policy.record(&retained).unwrap().unwrap(); + assert!(completed.retired); + assert_eq!(completed.metadata, Some(frozen)); + assert!(!journal.exists()); + assert_eq!( + snapshots + .retire(directory.to_str().unwrap(), retained) + .await + .unwrap(), + 0 + ); + assert_checkout_metadata_removed(&binding); + assert_protected(&data); + let session = data["session"].as_str().unwrap(); + let expected_sequence = data["session_sequence"].as_i64().unwrap(); + assert_eq!(sequence(&store, session), expected_sequence); + let app = build(&paths, &directory, &root).await; + let client = Client::embedded(app.embedded()).at(directory.to_str().unwrap()); + receipt_roundtrip(&client, session, &receipt).await; + assert_eq!(sequence(&store, session), expected_sequence); + assert_eq!( + std::fs::read_link(directory.join("tracked")).unwrap(), + owner + ); + std::fs::set_permissions(&owner, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!( + std::fs::read(owner).unwrap(), + b"owner after managed metadata process death" + ); + app.runtime.shutdown().await; +} +fn assert_cut_state( + before: &cyber_server::runtime::snapshot_reclamation::Record, + journal: &Path, + cut: &str, +) { + let frozen = before.metadata.as_ref().unwrap(); + assert!(before.completed && !before.retired && frozen.version == 2); + let binding = &frozen.checkouts[0].binding; + let activity = binding + .common_dir + .join("cyber-worktree-activity") + .join(&binding.worktree_id); + assert!( + !activity + .join(format!("{}.snapshot-ack", binding.owner_id)) + .exists() + ); + assert_eq!( + activity.join(format!("{}.lock", binding.owner_id)).exists(), + cut == "acknowledgement" + ); + assert!(journal.join("intent.json").exists()); +} + +fn assert_protected(data: &Value) { + for entry in data["protected"].as_array().unwrap() { + let path = entry["path"].as_str().unwrap(); + let bytes: Vec = serde_json::from_value(entry["bytes"].clone()).unwrap(); + assert_eq!(std::fs::read(path).unwrap(), bytes); + } +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 888b42ed3..27ad6fc93 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -144,8 +144,11 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. keep/rollback positives now remove those leaves and acknowledge from the retained receipt after restart without source replay. Guard reopen/cut, alias/context/live-owner/construction and bounded FIFO cases pass locally. - Actual managed metadata-deletion process cuts, zero-payload/partial constructors, - old/deleted Locations and source-bound platforms still gate M6 acceptance. + Ordinary managed metadata deletion now also passes actual process kills after + acknowledgement removal and after claim removal, followed by physical Store + reopen and periodic resumption of the same plan. Broader preparation/paired + deletion cuts, zero-payload/partial constructors, old/deleted Locations and + source-bound platforms still gate M6 acceptance. Native run [42733865471](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42733865471) passes Linux x64/ARM and macOS ARM/Intel at `e2b3ba18`, including the earlier diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 4386b6fe6..a06d7c8cb 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1283,3 +1283,44 @@ M6, parent tasks and issue #93 remain open; no PR, merge or delivery is claimed. The final OpenAPI artifact consistency test also passes (one exact HTTP case). No generated HTTP/SDK schema changes are required by this internal protocol. + +Managed metadata process-death continuation (2026-11-10): an actual App creates a +managed checkout in default application data, captures/restores through the real +owned worker, observes payload completion and starts metadata retirement. A +configuration callback observes the real acknowledgement unlink, or the following +original-claim unlink, and parks before the next native deletion. The parent +verifies readiness, sends SIGKILL and reaps the child. No native source/deletion +hook or Store event is replaced. Failed control/child fixtures remain retained. +The original 30-second restore confirmation bound is unchanged; a separate new +90-second parent deadline covers the complete fixture setup and selected cut. + +After each kill, a physically reopened Store shows version-2 metadata planning +without completion, the exact missing checkout leaves and the still-retained +intent. Production periodic maintenance resumes the frozen complete bindings, +finishes the same plan and returns zero on repeat. Protected Git index/config and +shared creation bytes, Session event sequence and a later source symlink to an +unreadable owner file stay unchanged. A fresh App returns the same receipt through +explicit acknowledgement after metadata is gone. These are ordinary managed +restore cuts at acknowledgement/claim removal, not the full managed preparation/ +paired/constructor/removed-Location crash matrix. The child entry point is inert +in ordinary test discovery and actually executes twice through its parent. + +The existing managed App fixture setup and tracked restore are shared with the +new subprocess case without changing its source/ownership assertions. Native CI +already runs the complete Git App binary; its path filter now explicitly includes +the new support file. This advances M6/M8 acceptance without closing either task. + +Final local cut verification passes all four Git App entries in 48.73 seconds, +including the inert child and parent exercising two actual SIGKILLs. The focused +new parent also passes independently in 28.83 seconds. App all-target Clippy with +warnings denied passes after splitting cut-state assertions from resumption (the +initial combined helper measured 18/15; no allowance was added). Formatting, +whitespace, strict OpenSpec (100 items) and spec lint (0 errors/25 warnings) pass. +Code-review-skill review checks the exact observed cut, unconditional child +kill/reap, retained failure roots, physical Store reopen, immutable plan equality, +production maintenance resumption and data-only acknowledgement assertions; no +additional issue is found in this test continuation. Native C sandbox and linker +unwind diagnostics were observed without assigning their origin. No production +behavior, wire/SDK contract, default Git admission or parent completion changes. +Full workspace/SDK runtime and fresh-source cross-platform proof were not performed +locally; the native matrix must execute this source before accepting M8. From d702c85a5315b020d93b0effb9ec6894eef6e7cb Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 05:01:34 -0400 Subject: [PATCH 69/87] Retire ordinary restore metadata without displaced bytes --- .../cyber-app/tests/snapshot_reclamation.rs | 20 ++- .../support/snapshot_native_retirement.rs | 133 ++++++++++++++++++ crates/cyber-snapshot/src/copy/lifetime.rs | 12 +- crates/cyber-snapshot/src/copy/reclamation.rs | 12 +- docs/goal-delivery.md | 18 +++ .../managed-recovery.md | 17 +++ .../complete-daily-coding-surfaces/tasks.md | 44 ++++++ 7 files changed, 232 insertions(+), 24 deletions(-) diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs index 9a80951fc..d829e0424 100644 --- a/crates/cyber-app/tests/snapshot_reclamation.rs +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -39,6 +39,9 @@ impl Fixture { Self::in_directory(tempfile::tempdir().unwrap()).await } async fn in_directory(temp: tempfile::TempDir) -> Self { + Self::with_current(temp, Some("after!\n".repeat(4096))).await + } + async fn with_current(temp: tempfile::TempDir, current: Option) -> Self { let root = temp.path().canonicalize().unwrap(); let paths = Paths { data: root.join("data"), @@ -101,11 +104,11 @@ impl Fixture { .unwrap() .to_owned(); for index in 0..2 { - std::fs::write( - project.join(format!("file_{index}")), - "after!\n".repeat(4096), - ) - .unwrap(); + let path = project.join(format!("file_{index}")); + match ¤t { + Some(bytes) => std::fs::write(path, bytes).unwrap(), + None => std::fs::remove_file(path).unwrap(), + } } prompt(&app, &client, &session, "after snapshot", 2).await; let mut selection = json!({"message_id":first,"target":"code","action":"commit","expected_directory":project}); @@ -137,7 +140,12 @@ impl Fixture { }) .find(|path| path.exists()) .unwrap(); - assert!(journal.join("stage_0").exists() && journal.join("stage_1").exists()); + let displaced = current + .as_ref() + .is_some_and(|bytes| bytes != &"before\n".repeat(4096)); + for index in 0..2 { + assert_eq!(journal.join(format!("stage_{index}")).exists(), displaced); + } let config = Arc::new(Mutex::new(json!({"snapshots":{"retention_days":7}}))); let snapshots = maintenance( &paths, diff --git a/crates/cyber-app/tests/support/snapshot_native_retirement.rs b/crates/cyber-app/tests/support/snapshot_native_retirement.rs index 3b817f179..41f36864c 100644 --- a/crates/cyber-app/tests/support/snapshot_native_retirement.rs +++ b/crates/cyber-app/tests/support/snapshot_native_retirement.rs @@ -26,6 +26,139 @@ fn worker_routes(f: &Fixture) -> [PathBuf; 2] { ] } +async fn zero_payload_retirement(current: Option, displaced_files: usize) { + let f = Fixture::with_current(tempfile::tempdir().unwrap(), current).await; + // Zero-byte plans still need the ordinary retention and native proof gates. + assert_eq!(f.reclaim().await.unwrap(), 0); + assert!(f.record().is_none()); + f.age(); + zero_payload_fences(&f).await; + assert_eq!(f.reclaim().await.unwrap(), 0); + let payload = f.record().expect("zero-byte cleanup must commit its plan"); + assert!(payload.completed && !payload.retired); + assert_eq!(payload.plan.files.len(), displaced_files); + assert_eq!(payload.plan.total_bytes().unwrap(), 0); + assert_zero_payload_retired(&f).await; + f.app.runtime.shutdown().await; +} + +async fn assert_zero_payload_retired(f: &Fixture) { + let requests = f.model.requests().len(); + let routes = worker_routes(f); + f.snapshots.gc().await; + assert!(f.record().unwrap().retired); + assert!(f.record().unwrap().metadata.unwrap().total_bytes().unwrap() > 0); + assert!(!f.journal.exists()); + assert!(routes.iter().all(|path| !path.exists())); + assert_eq!(retire(f).await.unwrap(), 0); + acknowledge_zero_payload(f).await; + assert_eq!(f.model.requests().len(), requests); +} + +async fn acknowledge_zero_payload(f: &Fixture) { + let reply = f + .client + .post( + &format!( + "/sessions/{}/rewind/restores/{}/acknowledge", + f.session, f.receipt.origin.activity_id + ), + json!({}), + ) + .await + .unwrap(); + assert_eq!( + serde_json::from_value::(reply["data"].clone()).unwrap(), + f.receipt + ); + for index in 0..2 { + assert_eq!( + std::fs::read(f.project.join(format!("file_{index}"))).unwrap(), + "before\n".repeat(4096).as_bytes() + ); + } +} + +async fn zero_payload_fences(f: &Fixture) { + for config in [ + Value::Null, + json!({"snapshots":false}), + json!({"snapshots":{"retention_days":0}}), + ] { + *f.config.lock().unwrap() = config; + assert_eq!(f.reclaim().await.unwrap(), 0); + assert!(f.record().is_none()); + } + *f.config.lock().unwrap() = json!({}); + let unknown = f.journal.join("unknown-construction"); + std::fs::write(&unknown, b"retain unknown evidence").unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(f.record().is_none()); + assert_eq!(std::fs::read(&unknown).unwrap(), b"retain unknown evidence"); + // Only remove the file this test just introduced, after proving refusal. + std::fs::remove_file(unknown).unwrap(); + let held = std::fs::File::open(&worker_routes(f)[0]).unwrap(); + held.try_lock().unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(f.record().is_none()); + drop(held); + let phase = &worker_routes(f)[1]; + let original = std::fs::read(phase).unwrap(); + let mut changed: Value = serde_json::from_slice(&original).unwrap(); + changed["admitted"] = (!changed["admitted"].as_bool().unwrap()).into(); + std::fs::write(phase, changed.to_string()).unwrap(); + assert!(f.reclaim().await.is_err()); + assert!(f.record().is_none()); + std::fs::write(phase, original).unwrap(); +} + +#[tokio::test] +async fn zero_payload_created_restore_retires_native_metadata() { + zero_payload_retirement(None, 0).await; +} + +#[tokio::test] +async fn zero_payload_empty_displaced_restore_retires_native_metadata() { + zero_payload_retirement(Some(String::new()), 2).await; +} + +#[tokio::test] +async fn zero_payload_unchanged_restore_retires_native_metadata() { + zero_payload_retirement(Some("before\n".repeat(4096)), 0).await; +} + +#[tokio::test] +async fn zero_payload_interrupted_completion_resumes_the_exact_plan() { + for after_delete in [false, true] { + let f = Fixture::with_current(tempfile::tempdir().unwrap(), None).await; + f.age(); + let cut = maintenance( + &f.paths, + f.config.clone(), + Arc::new(super::Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after_delete, + tamper: None, + pause: None, + }), + ); + assert!( + cut.reclaim(f.project.to_str().unwrap()) + .await + .unwrap_err() + .contains("injected cleanup interruption") + ); + let planned = f.record().unwrap(); + assert!(!planned.completed && planned.plan.files.is_empty()); + assert_eq!(f.reclaim().await.unwrap(), 0); + let completed = f.record().unwrap(); + assert!(completed.completed); + assert_eq!(completed.plan, planned.plan); + assert_zero_payload_retired(&f).await; + f.app.runtime.shutdown().await; + } +} + #[tokio::test] async fn native_retirement_removes_exact_metadata_and_acknowledges_from_the_retained_store_receipt() { diff --git a/crates/cyber-snapshot/src/copy/lifetime.rs b/crates/cyber-snapshot/src/copy/lifetime.rs index ce0d19295..c754322bc 100644 --- a/crates/cyber-snapshot/src/copy/lifetime.rs +++ b/crates/cyber-snapshot/src/copy/lifetime.rs @@ -252,14 +252,6 @@ impl Store { phase.checkout_bindings = Some(resources.to_vec()); self.write_source_phase(binding, &phase) } - pub(crate) fn verify_reclamation_phase( - &self, - binding: &SnapshotLifetimeBinding, - origin: &cyber_server::runtime::SnapshotOrigin, - resources_digest: &str, - ) -> Result<(), String> { - self.verify_reclamation_phase_admission(binding, origin, resources_digest, true) - } pub(crate) fn verify_reclamation_phase_admission( &self, binding: &SnapshotLifetimeBinding, @@ -670,11 +662,11 @@ mod tests { lifetime.admit_source(&origin).unwrap(); let digest = binding.checkout_resources_digest(&resources).unwrap(); store - .verify_reclamation_phase(&binding, &origin, &digest) + .verify_reclamation_phase_admission(&binding, &origin, &digest, true) .unwrap(); assert!( store - .verify_reclamation_phase(&binding, &origin, &"f".repeat(64)) + .verify_reclamation_phase_admission(&binding, &origin, &"f".repeat(64), true) .is_err() ); assert!( diff --git a/crates/cyber-snapshot/src/copy/reclamation.rs b/crates/cyber-snapshot/src/copy/reclamation.rs index 2e4b82a69..71325ca5b 100644 --- a/crates/cyber-snapshot/src/copy/reclamation.rs +++ b/crates/cyber-snapshot/src/copy/reclamation.rs @@ -93,10 +93,6 @@ fn ordinary(record: &Journal) -> bool { && record.resolution.is_none() && record.settled_by.is_none() && record.context.is_some() - && record - .changes - .iter() - .any(|change| change.before.as_ref().is_some_and(|entry| entry.length > 0)) } /// Read existing metadata only. A missing/partial namespace remains protected. pub(crate) fn locations(data: &Path) -> Result, String> { @@ -185,9 +181,8 @@ pub(crate) fn reclaim( store, directory, record, settlement, frozen, workspace, config, )?; let plan = prepared.plan.clone(); - if plan.files.is_empty() { - return Ok(0); - } + // Empty inventories still commit native verification/completion so that + // receipt-bound metadata retirement can subsequently reclaim this owner. policy.reclaim(plan, Box::new(move || prepared.delete())) } impl Store { @@ -305,10 +300,11 @@ fn prepare( .as_deref() .ok_or("Missing reclamation source phase")?; let phase = MetadataPin::open(&workers, name)?; - store.verify_reclamation_phase( + store.verify_reclamation_phase_admission( binding, plan.receipt.origin().map_err(storage::fail)?, plan.receipt.owners().map_err(storage::fail)?[0].resources_digest, + !record.changes.is_empty(), )?; phase.verify(&workers, name)?; storage::existing_child( diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 27ad6fc93..8fe47c22e 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -27,6 +27,24 @@ section 5/6 implementation state; reconcile it at delivery. | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | | #92 protocol/callbacks/producers, tasks 3–6/S/D | Finish contribution ownership, hooks/providers/catalog transforms and TypeScript kit, durable scoped callbacks (`session/prompt` still explicitly refuses), Setup and DirectoryAdded plus combined producer acceptance. Preserve accepted FileChanged and CwdChanged fixes. Complete S1 epoch/directory-key/legacy-deletion and D1 directory-admission/Safe Boundary/child-policy review before their dependent code. | +### Zero-byte ordinary restore continuation + +Ordinary cleanup now records payload completion for exact empty inventories and +zero-length displaced files, allowing periodic maintenance to retire their native +metadata. Actual App regressions cover restored file creation, displaced empty +files and unchanged restores; all three failed before the fix and pass afterward. +An unchanged restore requires its non-admitted phase, while changed restores still +require source admission. Unknown construction, live owner locks, contradictory +phase evidence and unavailable/disabled/zero-retention policy remain fences. +Paired/preparation zero-payload paths, construction and removed-Location acceptance +remain required; no parent task or delivery is complete from this increment. + +Native run [42774849027](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42774849027) +at `b42f5fca` completed successfully on Linux x64/ARM and macOS ARM/Intel. This is +source-bound evidence for that checkpoint, not acceptance of later managed +retirement/process-cut or zero-byte changes. Main was fetched for this continuation; +`8266db6c` remains fully integrated. + ### Focused code-review findings and next actions This review covers incoming JSONL behavior and merge overlap, not acceptance of diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 2b75ce39c..453373a68 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -1046,6 +1046,23 @@ construction entries for the same owner remain fenced; unrelated activity and shared creation records remain intact. The callback remains serialized against reference writers, with fresh retention and complete native-owner re-attestation. +### Ordinary restores without displaced bytes + +Acknowledged ordinary restores SHALL use the same bounded payload plan/completion +protocol when their exact native inventory contains no leaves or only zero-length +leaves. Restoring an absent source file and an unchanged restore SHALL NOT leave +metadata permanently ineligible for retirement. A zero byte count alone is not +completion: the native callback SHALL validate the complete journal, immutable +intent, ownership, consumed checkout resources, exited lifetime and source phase, +and synchronize the journal before the Store commits completion. An unchanged +restore SHALL require its positively non-admitted source phase; a restore with +changes SHALL require its admitted phase. Missing expected leaves without an +existing exact plan, unknown construction and contradictory admission still refuse. +Retention, reference and namespace fences apply before these plans as usual. +Periodic maintenance MAY then retire the exact metadata through the existing +receipt-bound protocol. Paired and cancelled preparation zero-payload acceptance +remain separate required paths. + ### Paired recovery payload cleanup Acknowledged keep/rollback cleanup SHALL bind the exact durable two-owner recovery diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index a06d7c8cb..0abfa65f1 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1324,3 +1324,47 @@ unwind diagnostics were observed without assigning their origin. No production behavior, wire/SDK contract, default Git admission or parent completion changes. Full workspace/SDK runtime and fresh-source cross-platform proof were not performed locally; the native matrix must execute this source before accepting M8. + +Ordinary zero-payload continuation: remove the positive displaced-byte eligibility +requirement and let an exact empty native inventory run through payload planning, +the serialized native verification callback and durable completion. Empty displaced +files remain actual leaves that must be verified and unlinked despite reporting +zero bytes. Unchanged restores require their positively non-admitted source phase; +restores with changes still require admission. The same immutable receipt, intent, +checkout, lifetime, namespace, reference and retention checks apply. Remove the +now-unused admitted-only phase wrapper and keep its resource/digest unit tests on +the explicit admission verifier. Paired/preparation zero-inventory paths remain +separate; this does not claim their completion. + +Three actual App cases (restored creation, empty displaced files and unchanged +restore) failed before the fix because no cleanup plan was committed, then pass +with periodic metadata retirement and data-only HTTP receipt acknowledgement. +Each also refuses unavailable/disabled/zero retention, unknown construction, a +live lifetime lock and contradictory source admission before committing a plan. +A fourth regression interrupts the zero-leaf callback before and after native +verification and resumes the exact durable plan before periodic retirement. +Final focused selection passes four tests in 25.24 seconds. +The full reclamation selection before the final assertion-helper extraction and +fourth test addition passes 41 cases in 251.50 seconds; the final focused run +verifies the extracted helpers and added interruption case. Native macOS C sandbox +deprecation and linker unwind diagnostics were observed without assigning origin. +Full workspace/SDK runtime and fresh-source platform checks were not run locally. +Lifetime tests pass +seven cases with one explicit subprocess fixture ignored. App/snapshot all-target +Clippy with warnings denied passes after splitting cleanup and acknowledgement +assertions (the original combined helper measured 24/15; no allowance added). +Strict OpenSpec passes 100 items and spec lint reports 0 errors/25 warnings. + +Focused code-review-skill review finds no outstanding issue in the changed native +admission and fixture paths: empty plans still execute full pre/post verification +and synchronization, missing expected images still require an exact frozen plan, +and periodic retirement cannot precede completion. Architecture follows the +existing native adapter/Store-policy port; no new HTTP, SDK or schema surface is +introduced. Cross-platform acceptance, managed/Git zero-payload cases, construction, +old/removed Location and complete crash/client acceptance remain required. M6/M8, +all parent tasks and issue #93 remain open. + +Native run 42774849027 at b42f5fca now has four successful platform jobs, including +macOS Intel. Later managed-retirement and real-cut checkpoints are running/queued; +that success does not accept later source. Fetching main finds 8266db6c already +integrated, with no new upstream commits to merge. From af0d10edb94ff07959ccbcdf6a4ba88c27506020 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 05:22:44 -0400 Subject: [PATCH 70/87] Retire empty preparation and recovery journals --- .github/workflows/non-git-snapshot-proof.yml | 26 +- crates/cyber-cli/tests/branches_cli.rs | 3 + .../cyber-cli/tests/support/zero_payload.rs | 367 ++++++++++++++++++ .../src/copy/reclamation/cancelled.rs | 3 - .../src/copy/reclamation/paired.rs | 6 +- docs/goal-delivery.md | 18 +- .../managed-recovery.md | 15 +- .../complete-daily-coding-surfaces/tasks.md | 54 +++ 8 files changed, 475 insertions(+), 17 deletions(-) create mode 100644 crates/cyber-cli/tests/support/zero_payload.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 16ea63161..1afd0cf8e 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -78,8 +78,10 @@ jobs: set -euo pipefail cargo test --locked -p cyber-core --lib worktrees::activity::snapshot::acknowledgement::tests::maintenance_requires_an_existing_acknowledgement_without_publishing_one -- --exact --nocapture 2>&1 | tee managed-checkout-ack-proof.log - name: Owned Git runtime acceptance candidate + # App proxy sockets include the Session ID. Short roots fit Linux + # sockaddr_un; the same root is retained by the artifact step below. env: - TMPDIR: ${{ runner.temp }}/snapshot-fixtures + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail @@ -89,7 +91,7 @@ jobs: - name: Exact acknowledged checkout retirement and resumption if: ${{ !cancelled() }} env: - TMPDIR: ${{ runner.temp }}/snapshot-fixtures + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail @@ -102,7 +104,7 @@ jobs: cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_reclamation -- --nocapture 2>&1 | tee fallback-collection-proof.log - name: Managed Git CLI process-death recovery and cancellation candidate env: - TMPDIR: ${{ runner.temp }}/snapshot-fixtures + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail @@ -111,7 +113,7 @@ jobs: - name: Pre-effect preparation and TUI lost-reply reconciliation if: ${{ !cancelled() }} env: - TMPDIR: ${{ runner.temp }}/snapshot-fixtures + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail @@ -120,7 +122,7 @@ jobs: - name: Prepared cancellation metadata retirement if: ${{ !cancelled() }} env: - TMPDIR: ${{ runner.temp }}/snapshot-fixtures + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail @@ -129,12 +131,21 @@ jobs: - name: Paired metadata retirement after native source kill if: ${{ !cancelled() }} env: - TMPDIR: ${{ runner.temp }}/snapshot-fixtures + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail mkdir -p "$TMPDIR" cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli managed_keep_and_rollback_after_real_partial_restore_kill_preserve_owner_files_and_never_replay -- --exact --nocapture --test-threads=1 2>&1 | tee paired-retirement-proof.log + - name: Zero-payload preparation and paired retirement after native cuts + if: ${{ !cancelled() }} + env: + TMPDIR: /tmp/c${{ github.run_number }} + shell: bash + run: | + set -euo pipefail + mkdir -p "$TMPDIR" + cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli zero_payload:: -- --nocapture --test-threads=1 2>&1 | tee zero-payload-retirement-proof.log - name: JSONL driver and atomic last-message regressions if: ${{ !cancelled() }} shell: bash @@ -164,6 +175,7 @@ jobs: preparation-client-proof.log preparation-retirement-proof.log paired-retirement-proof.log + zero-payload-retirement-proof.log jsonl-driver-proof.log structured-result-proof.log if-no-files-found: warn @@ -172,6 +184,6 @@ jobs: uses: actions/upload-artifact@v4 with: name: snapshot-failed-cli-fixtures-${{ matrix.os }} - path: ${{ runner.temp }}/snapshot-fixtures + path: /tmp/c${{ github.run_number }} include-hidden-files: true if-no-files-found: warn diff --git a/crates/cyber-cli/tests/branches_cli.rs b/crates/cyber-cli/tests/branches_cli.rs index b13fb1276..2d566d528 100644 --- a/crates/cyber-cli/tests/branches_cli.rs +++ b/crates/cyber-cli/tests/branches_cli.rs @@ -15,6 +15,9 @@ use serde_json::{Value, json}; mod reclamation_support; #[path = "../../cyber-app/tests/support/scripted_model.rs"] mod scripted_model; +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[path = "support/zero_payload.rs"] +mod zero_payload; /// `WRITE-A` and `WRITE-B` write a file, `GATE` waits at the model, and anything else is /// answered with text. diff --git a/crates/cyber-cli/tests/support/zero_payload.rs b/crates/cyber-cli/tests/support/zero_payload.rs new file mode 100644 index 000000000..f7ec4e7d4 --- /dev/null +++ b/crates/cyber-cli/tests/support/zero_payload.rs @@ -0,0 +1,367 @@ +//! Real native cuts for empty cancellation and paired cleanup inventories. +use super::*; +use cyber_server::runtime::{ + SnapshotPreparationReceipt, SnapshotRecoveryReceipt, snapshot_reclamation::Receipt, +}; +const FILES: usize = 256; + +struct Fixture { + model: scripted_model::ScriptedModel, + env: Env, + session: String, + first: String, + messages: Vec, +} +impl Fixture { + fn copy(created: bool) -> Self { + let model = scripted_model::ScriptedModel::start(reply); + let env = Env::new(&model); + std::fs::remove_dir_all(env.directory.join(".git")).unwrap(); + Self::new(model, env, None, created) + } + #[cfg(feature = "native-git-acceptance")] + fn managed(created: bool) -> Self { + let model = scripted_model::ScriptedModel::start(reply); + let (env, session) = managed_git_fixture(&model); + Self::new(model, env, Some(session), created) + } + fn new( + model: scripted_model::ScriptedModel, + env: Env, + session: Option, + created: bool, + ) -> Self { + if created { + write_files(&env); + } + let session = env.exec(session.as_deref(), "before snapshot"); + let first = env.user_ids(&session)[0].clone(); + if created { + for index in 0..FILES { + std::fs::remove_file(env.directory.join(format!("bulk_{index:04}"))).unwrap(); + } + } else { + write_files(&env); + } + env.exec(Some(&session), "after snapshot"); + let messages = env.messages(&session); + Self { + model, + env, + session, + first, + messages, + } + } +} +fn write_files(env: &Env) { + for index in 0..FILES { + std::fs::write(env.directory.join(format!("bulk_{index:04}")), "before").unwrap(); + } +} + +#[test] +fn copy_cancelled_deletion_preparation_retires_zero_payload() { + cancelled(Fixture::copy(false)); +} +#[test] +fn copy_kept_creation_after_completed_source_kill_retires_zero_payload() { + kept(Fixture::copy(true)); +} +#[cfg(feature = "native-git-acceptance")] +#[test] +fn managed_cancelled_deletion_preparation_retires_zero_payload() { + cancelled(Fixture::managed(false)); +} +#[cfg(feature = "native-git-acceptance")] +#[test] +fn managed_kept_creation_after_completed_source_kill_retires_zero_payload() { + kept(Fixture::managed(true)); +} + +fn cancelled(f: Fixture) { + let git = preparation_git_metadata(&f.env.directory); + let activity = kill_after_prepared_images(&f.env, &f.session, &f.first); + let review = json(&f.env.cyber(&[ + "sessions", + "recovery-preparation", + &f.session, + &activity, + "--format", + "json", + ])); + let body = json!({"confirmation":review["confirmation"],"request_id":"empty-preparation"}); + let reply = json(&preparation_confirm_cli( + &f.env, &f.session, &activity, &body, + )); + let receipt: SnapshotPreparationReceipt = serde_json::from_value(reply.clone()).unwrap(); + let requests = f.model.requests().len(); + retire_empty(&f, Receipt::Preparation(Box::new(receipt)), activity); + assert_eq!( + json(&preparation_request_cli( + &f.env, + &f.session, + reply["activity_id"].as_str().unwrap(), + "empty-preparation", + true + )), + reply + ); + assert_owner_state(&f, requests, &git); +} + +fn kept(f: Fixture) { + let git = preparation_git_metadata(&f.env.directory); + kill_completed_creation(&f); + let (journal, review) = completed_review(&f); + let request = json!({"journal_id":journal,"action":"keep","confirmation":review["confirmation"],"request_id":"empty-keep"}); + let reply = json(&recovery_confirm_cli(&f.env, &f.session, &request)); + assert_eq!(reply["journal_acknowledged"], true); + let receipt: SnapshotRecoveryReceipt = + serde_json::from_value(reply["receipt"].clone()).unwrap(); + let owner = receipt + .checkout + .as_ref() + .unwrap() + .resolution_origin + .activity_id + .clone(); + let requests = f.model.requests().len(); + retire_empty(&f, Receipt::Recovery(receipt), owner); + // Lookup starts a fresh service; explicit acknowledgement uses retained history. + let lookup = json(&f.env.cyber(&[ + "sessions", + "recovery-request", + &f.session, + "empty-keep", + "--acknowledge", + "--format", + "json", + ])); + assert_eq!(lookup["receipt"], reply["receipt"]); + assert_owner_state(&f, requests, &git); +} + +fn completed_review(f: &Fixture) -> (String, Value) { + let catalog = json( + &f.env + .cyber(&["sessions", "recovery", &f.session, "--format", "json"]), + ); + let entries = catalog.as_array().unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0]["filesystem_complete"], true); + let journal = entries[0]["id"].as_str().unwrap().to_owned(); + let review = json(&f.env.cyber(&[ + "sessions", "recovery", &f.session, &journal, "--action", "keep", "--format", "json", + ])); + assert!( + review["conflicts"].as_array().unwrap().is_empty(), + "{review}" + ); + (journal, review) +} + +fn retire_empty(f: &Fixture, receipt: Receipt, owner: String) { + use std::os::unix::fs::PermissionsExt; + exits(&f.env.cyber(&["service", "stop"]), 0); + let maintenance = reclamation_support::Maintenance::open(&f.env.root.join("cyber-home")); + assert_eq!(maintenance.reclaim(&f.env.directory).unwrap(), 0); + assert!(maintenance.policy.record(&receipt).unwrap().is_none()); + maintenance.age(owner); + let later = f.env.root.join("later-owner"); + std::fs::write(&later, b"later owner bytes").unwrap(); + std::fs::remove_file(f.env.directory.join("bulk_0000")).unwrap(); + std::os::unix::fs::symlink(&later, f.env.directory.join("bulk_0000")).unwrap(); + std::fs::set_permissions(&later, std::fs::Permissions::from_mode(0o0)).unwrap(); + // Refuse unknown construction even when the expected inventory is empty. + let journal = maintenance + .namespace + .join("journals") + .join(&receipt.origin().unwrap().binding.journal_id); + let unknown = journal.join("unknown-construction"); + std::fs::write(&unknown, b"retain unknown bytes").unwrap(); + assert!(maintenance.reclaim(&f.env.directory).is_err()); + assert!(maintenance.policy.record(&receipt).unwrap().is_none()); + assert_eq!(std::fs::read(&unknown).unwrap(), b"retain unknown bytes"); + std::fs::remove_file(unknown).unwrap(); + assert_empty_plan(f, &maintenance, &receipt); + assert!( + maintenance + .retire(&f.env.directory, receipt.clone()) + .unwrap() + > 0 + ); + let record = maintenance.policy.record(&receipt).unwrap().unwrap(); + assert!(record.retired); + let checkouts = record.metadata.unwrap().checkouts; + let managed = f + .env + .directory + .starts_with(f.env.root.join("cyber-home/data/worktrees")); + let owners = receipt.owners().unwrap(); + assert_eq!(checkouts.len(), if managed { owners.len() } else { 0 }); + assert_retired_checkout_files(&checkouts); + assert_retired_worker_files(&maintenance.namespace, &receipt); + for owner in receipt.owners().unwrap() { + assert!( + !maintenance + .namespace + .join("journals") + .join(&owner.origin.binding.journal_id) + .exists() + ); + } + assert_eq!(maintenance.retire(&f.env.directory, receipt).unwrap(), 0); +} +fn assert_empty_plan( + f: &Fixture, + maintenance: &reclamation_support::Maintenance, + receipt: &Receipt, +) { + assert!( + maintenance + .plan_only(&f.env.directory) + .unwrap_err() + .contains("fixture cut after durable paired plan") + ); + let pending = maintenance.policy.record(receipt).unwrap().unwrap(); + assert!(!pending.completed && pending.plan.files.is_empty()); + assert_eq!(maintenance.reclaim(&f.env.directory).unwrap(), 0); + let complete = maintenance.policy.record(receipt).unwrap().unwrap(); + assert!(complete.completed); + assert_eq!(complete.plan, pending.plan); + assert_eq!(complete.plan.total_bytes().unwrap(), 0); +} +fn assert_owner_state(f: &Fixture, requests: usize, git: &[(PathBuf, Vec)]) { + use std::os::unix::fs::PermissionsExt; + let later = f.env.root.join("later-owner"); + assert_eq!( + std::fs::read_link(f.env.directory.join("bulk_0000")).unwrap(), + later + ); + assert_eq!( + std::fs::metadata(&later).unwrap().permissions().mode() & 0o777, + 0 + ); + std::fs::set_permissions(&later, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!(std::fs::read(&later).unwrap(), b"later owner bytes"); + for index in 1..FILES { + assert_eq!( + f.env.file(&format!("bulk_{index:04}")).as_deref(), + Some("before") + ); + } + for (path, bytes) in git { + assert_eq!(&std::fs::read(path).unwrap(), bytes); + } + assert_eq!(f.env.messages(&f.session), f.messages); + assert_eq!(f.model.requests().len(), requests); +} + +fn kill_completed_creation(f: &Fixture) { + use rusqlite::OptionalExtension; + let selection = reviewed_selection(&f.env, &f.first, "both", "commit"); + let preview = reviewed_preview(&f.env, &f.session, &selection); + let mut body = selection; + body["confirmation"] = preview["confirmation"].clone(); + let mut pending = Confirm( + f.env + .command(&[ + "api", + "POST", + &format!("/sessions/{}/rewind/confirm", f.session), + "-d", + &body.to_string(), + ]) + .stdout(std::fs::File::create(f.env.root.join("zero-confirm.stdout")).unwrap()) + .stderr(std::fs::File::create(f.env.root.join("zero-confirm.stderr")).unwrap()) + .spawn() + .unwrap(), + ); + let deadline = std::time::Instant::now() + Duration::from_secs(120); + while f.env.file("bulk_0000").as_deref() != Some("before") { + assert!( + pending.0.try_wait().unwrap().is_none(), + "restore exited before source creation" + ); + assert!( + std::time::Instant::now() < deadline, + "first source creation was not observed" + ); + std::thread::sleep(Duration::from_millis(1)); + } + let database = std::fs::read_dir(f.env.root.join("cyber-home/data")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|ext| ext == "db")) + .unwrap(); + let db = rusqlite::Connection::open(database).unwrap(); + db.busy_timeout(Duration::from_secs(5)).unwrap(); + db.execute_batch("BEGIN IMMEDIATE").unwrap(); + let origin: Option = db.query_row("SELECT json_extract(data,'$.snapshot_origin') FROM event e WHERE type='native.activity.changed.1' AND json_extract(data,'$.session_id')=?1 AND json_extract(data,'$.snapshot_source_admitted')=1 AND json_extract(data,'$.snapshot_restore_receipt') IS NULL AND NOT EXISTS(SELECT 1 FROM event n WHERE n.aggregate_id=e.aggregate_id AND n.seq>e.seq)", [&f.session], |row| row.get(0)).optional().unwrap(); + let origin: Value = + serde_json::from_str(&origin.expect("writer cut missed native completion boundary")) + .unwrap(); + let intent = std::fs::read_dir(f.env.root.join("cyber-home/data/snapshot/global")) + .unwrap() + .map(|entry| { + entry + .unwrap() + .path() + .join("journals") + .join(origin["binding"]["journal_id"].as_str().unwrap()) + .join("intent.json") + }) + .find(|path| path.exists()) + .unwrap(); + wait_completed_intent(&f.env, &intent, &mut pending.0, deadline); + kill_registered_service(&f.env); + assert!(!pending.0.wait().unwrap().success()); + db.execute_batch("ROLLBACK").unwrap(); + assert_eq!( + f.env.file(&format!("bulk_{:04}", FILES - 1)).as_deref(), + Some("before") + ); +} +struct Confirm(std::process::Child); +impl Drop for Confirm { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} +fn wait_completed_intent( + env: &Env, + intent: &std::path::Path, + pending: &mut std::process::Child, + deadline: std::time::Instant, +) { + loop { + let record = std::fs::read(intent) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .unwrap_or(Value::Null); + if record["completed"] == true { + assert!( + record + .get("acknowledged") + .is_none_or(|value| value == false) + ); + assert_eq!( + std::fs::read_dir(intent.parent().unwrap()).unwrap().count(), + 2 + ); + break; + } + assert!( + pending.try_wait().unwrap().is_none(), + "restore exited before completed intent: {}", + restore_preparation_status(env) + ); + assert!( + std::time::Instant::now() < deadline, + "completed intent was not observed" + ); + std::thread::sleep(Duration::from_millis(2)); + } +} diff --git a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs index 8fd1c6c6d..772d1af01 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/cancelled.rs @@ -65,9 +65,6 @@ pub(super) fn reclaim( settlement, existing.map(|record| record.plan), )?; - if prepared.plan.files.is_empty() { - return Ok(0); - } policy.reclaim(prepared.plan.clone(), Box::new(move || prepared.delete())) } fn prepare( diff --git a/crates/cyber-snapshot/src/copy/reclamation/paired.rs b/crates/cyber-snapshot/src/copy/reclamation/paired.rs index 1962e60e8..31ae15138 100644 --- a/crates/cyber-snapshot/src/copy/reclamation/paired.rs +++ b/crates/cyber-snapshot/src/copy/reclamation/paired.rs @@ -59,9 +59,6 @@ pub(super) fn reclaim( settlement, existing.map(|record| record.plan), )?; - if prepared.plan.files.is_empty() { - return Ok(0); - } policy.reclaim(prepared.plan.clone(), Box::new(move || prepared.delete())) } @@ -423,6 +420,9 @@ impl PreparedPair { generation = ownership::generation(&directory)?; } } + // Empty or already removed inventories still need a durable native + // completion boundary for every owner before metadata retirement. + storage::sync(&owner.journal)?; } self.verify()?; self.plan.total_bytes().map_err(storage::fail) diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 8fe47c22e..f05d65a83 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -36,8 +36,13 @@ files and unchanged restores; all three failed before the fix and pass afterward An unchanged restore requires its non-admitted phase, while changed restores still require source admission. Unknown construction, live owner locks, contradictory phase evidence and unavailable/disabled/zero-retention policy remain fences. -Paired/preparation zero-payload paths, construction and removed-Location acceptance -remain required; no parent task or delivery is complete from this increment. +Cancelled deletion-only preparations and kept creations killed after filesystem +completion now use the same empty-plan protocol. Four real CLI cases cover both +file-copy and managed Git ownership, durable plan interruption/resumption, exact +claim/worker retirement and receipt acknowledgement after source replacement by +an unreadable later-owner symlink. Construction, removed-Location and complete +crash/platform acceptance remain required; no parent task or delivery is complete +from these increments. Native run [42774849027](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42774849027) at `b42f5fca` completed successfully on Linux x64/ARM and macOS ARM/Intel. This is @@ -45,6 +50,15 @@ source-bound evidence for that checkpoint, not acceptance of later managed retirement/process-cut or zero-byte changes. Main was fetched for this continuation; `8266db6c` remains fully integrated. +Run 42784209997 at `991b1519` subsequently fails the managed Git App setup on both +Linux platforms: the retained fixture root makes the sandbox proxy's Unix socket +path exceed the native address limit. The fixture root is now shortened under +`/tmp/c`, including nested process-cut fixtures, with the +artifact upload following the same root. No network policy or production socket +code changes. The workflow also runs the four zero-payload CLI cases and retains +their log. This resolves the inspected fixture routing problem in configuration; +fresh-source native execution is still required to prove acceptance. + ### Focused code-review findings and next actions This review covers incoming JSONL behavior and merge overlap, not acceptance of diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 453373a68..554224baa 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -1060,8 +1060,19 @@ changes SHALL require its admitted phase. Missing expected leaves without an existing exact plan, unknown construction and contradictory admission still refuse. Retention, reference and namespace fences apply before these plans as usual. Periodic maintenance MAY then retire the exact metadata through the existing -receipt-bound protocol. Paired and cancelled preparation zero-payload acceptance -remain separate required paths. +receipt-bound protocol. + +Acknowledged paired recovery and exactly cancelled preparations SHALL also commit +payload planning and observed completion for a verified empty inventory. A cancelled +deletion-only preparation has no prepared target leaves; its source remains +non-admitted. Keeping an owned creation after filesystem completion but before +Store settlement can leave neither original nor resolution payload leaves; both +intents, consumed owners, settlement links and their distinct admission states +remain required. Empty inventories SHALL execute the native verification callback +and synchronize every participating journal before completion. Zero bytes SHALL +NOT be interpreted as absence of a plan or permission to skip native proof. +Interrupted completion SHALL resume only the exact committed plan, with all +metadata retirement fences unchanged. ### Paired recovery payload cleanup diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 0abfa65f1..51a79004a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1368,3 +1368,57 @@ Native run 42774849027 at b42f5fca now has four successful platform jobs, includ macOS Intel. Later managed-retirement and real-cut checkpoints are running/queued; that success does not accept later source. Fetching main finds 8266db6c already integrated, with no new upstream commits to merge. + +Paired/preparation zero-payload continuation: both native adapters now submit a +verified empty inventory to the existing Store planning/completion protocol instead +of returning before the callback. Paired cleanup synchronizes every owner journal +even for an empty or already removed inventory. No receipt, event or API schema +changes; all owner, intent, acknowledgement, source-phase and reference fences +remain in force. This extends the approved M6 protocol rather than accepting M6. + +Four new real CLI regressions cover file-copy and managed Git deletion-only +preparation cancellation and kept creation after native completion but before +Store commit. Preparation uses the existing native kill helper. The completed- +source cut observes the first creation, holds the actual SQLite writer, observes +the complete native intent and empty private payload inventory, then kills the +service. Its client is always killed/reaped on scope exit. The new fixtures use +256 leaves to keep the cut observable; existing 1,280-leaf partial-cut fixtures and +their deadlines/assertions stay unchanged. Each case refuses unknown construction, +interrupts after committing an empty plan, resumes that exact plan, retires the +expected managed claim/acknowledgement count and all consumed worker/journal leaves, +and repeats retirement with zero effect. A later source symlink to unreadable owner +bytes, Git metadata, transcript and inference count survive cleanup and explicit +receipt acknowledgement after service restart. All four reached the empty-plan +shortcut before the production fix and failed there. The final native-feature +selection passes four cases in 47.43 seconds, including the complete managed +checkout-count assertions. Earlier fixture assumptions about omitted false fields +and partial-only recovery reviews were corrected before this regression result; +failed roots remain preserved. + +Native run 42784209997 at 991b1519 reports Git App setup failures on both Linux +platforms: retained fixture paths exceed the sandbox proxy Unix socket address +limit. The log from Linux x64 is retained locally for inspection. CI now uses a +short `/tmp/c` fixture root and uploads that same root, +including hidden files and nested managed process-cut fixtures. A new independent +step runs all four zero-payload CLI cases and retains its log. YAML parsing and +fixture/artifact root consistency pass. Production networking and sandbox policy +are unchanged; fresh-source CI must verify this routing correction. + +Default and native-feature CLI/snapshot all-target Clippy with warnings denied pass. Strict +OpenSpec passes 100 items and spec lint reports 0 errors/25 warnings. The first +green build attempt failed linking because the disk was full; only regenerable +Rust incremental compiler cache was cleared before a successful retry, preserving +source and native failure fixtures. Native C deprecation and linker unwind +diagnostics were observed without assigning their origin. Focused code-review-skill +review checks the empty callback's full pre/post verification, per-owner directory +synchronization, exact frozen plan resume and managed claim cardinality; no +outstanding issue is found in this continuation. This focused review approves the +continuation with zero findings; whole-branch review remains required for delivery. +The existing managed Git selection +passes three tests in 162.81 seconds: the deliberately caught fixture-preservation +probe, real partial-kill keep/rollback, and prepared cancellation with file/link +cleanup. The two file-copy zero-payload cases also pass in the default build in +16.87 seconds. Main is fetched again with no upstream commits missing from this +branch. Full workspace/SDK runtime and current-source platform +acceptance remain required. Constructor, old/removed Location and remaining native +cut/client acceptance stay open with M6/M8 and all parent tasks. From 2fb4183249b91e85e48c3c463c024d8a241f0adb Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 05:47:42 -0400 Subject: [PATCH 71/87] Verify reclamation across unavailable Location routes --- .github/workflows/non-git-snapshot-proof.yml | 5 + .../cyber-app/tests/snapshot_reclamation.rs | 120 +++++--- .../tests/support/location_retirement.rs | 290 ++++++++++++++++++ .../support/snapshot_native_retirement.rs | 18 ++ docs/goal-delivery.md | 13 + .../managed-recovery.md | 13 + .../specs/snapshots-checkpoints/spec.md | 9 + .../complete-daily-coding-surfaces/tasks.md | 53 ++++ 8 files changed, 481 insertions(+), 40 deletions(-) create mode 100644 crates/cyber-app/tests/support/location_retirement.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 1afd0cf8e..4b7f5d0b5 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -8,6 +8,7 @@ on: - crates/cyber-app/tests/support/scripted_model.rs - crates/cyber-app/tests/support/snapshot*.rs - crates/cyber-app/tests/support/managed_retirement_cut.rs + - crates/cyber-app/tests/support/location_retirement.rs - crates/cyber-server/src/runtime/snapshot*.rs - crates/cyber-server/src/runtime/snapshot_reclamation/** - crates/cyber-server/src/runtime/rewind*.rs @@ -98,9 +99,13 @@ jobs: mkdir -p "$TMPDIR" cargo test --locked -p cyber-core --lib worktrees::activity::snapshot::retirement:: -- --nocapture 2>&1 | tee checkout-retirement-proof.log - name: Store-bound fallback collection and reclamation + if: ${{ !cancelled() }} + env: + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail + mkdir -p "$TMPDIR" cargo test --locked -p cyber-app --features native-git-acceptance --test snapshot_reclamation -- --nocapture 2>&1 | tee fallback-collection-proof.log - name: Managed Git CLI process-death recovery and cancellation candidate env: diff --git a/crates/cyber-app/tests/snapshot_reclamation.rs b/crates/cyber-app/tests/snapshot_reclamation.rs index d829e0424..f82be4b5c 100644 --- a/crates/cyber-app/tests/snapshot_reclamation.rs +++ b/crates/cyber-app/tests/snapshot_reclamation.rs @@ -1,5 +1,7 @@ #![cfg(any(target_os = "macos", target_os = "linux"))] //! Real App restores/receipts and native private payloads; no source writes by cleanup. +#[path = "support/location_retirement.rs"] +mod location_retirement; #[path = "support/snapshot_native_retirement.rs"] mod native_retirement; #[path = "support/snapshot_retirement.rs"] @@ -34,6 +36,17 @@ struct Fixture { config: Arc>, snapshots: Arc, } +impl Drop for Fixture { + fn drop(&mut self) { + if std::thread::panicking() { + self._temp.disable_cleanup(true); + eprintln!( + "Preserved failed reclamation fixture at {}", + self._temp.path().display() + ); + } + } +} impl Fixture { async fn new() -> Self { Self::in_directory(tempfile::tempdir().unwrap()).await @@ -71,6 +84,71 @@ impl Fixture { }) .await .unwrap(); + let Restore { + client, + session, + receipt, + journal, + } = Restore::new(&app, &paths, &project, current).await; + let config = Arc::new(Mutex::new(json!({"snapshots":{"retention_days":7}}))); + let snapshots = maintenance( + &paths, + config.clone(), + cyber_app::snapshot_reclamation::policy(app.store.clone()), + ); + Self { + _temp: temp, + app, + client, + model, + paths, + project, + session, + receipt, + journal, + config, + snapshots, + } + } + fn age(&self) { + let owner = self.receipt.origin.activity_id.clone(); + // Only the retention clock is aged. Receipt/native ownership bytes and + // original event sequence remain those of the actual committed restore. + self.app + .store + .transaction(move |db| { + db.execute( + "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", + rusqlite::params![8i64 * 86400000, owner], + )?; + Ok(()) + }) + .unwrap(); + } + async fn reclaim(&self) -> Result { + self.snapshots.reclaim(self.project.to_str().unwrap()).await + } + fn record(&self) -> Option { + let receipt = self.receipt.clone(); + self.app + .store + .read(move |db| records::lookup(db, &receipt)) + .unwrap() + } +} +struct Restore { + client: Client, + session: String, + receipt: SnapshotRestoreReceipt, + journal: PathBuf, +} +impl Restore { + async fn new( + app: &App, + paths: &Paths, + project: &std::path::Path, + current: Option, + ) -> Self { let client = Client::embedded(app.embedded()).at(project.to_str().unwrap()); let session = client .post( @@ -89,7 +167,7 @@ impl Fixture { ) .unwrap(); } - prompt(&app, &client, &session, "before snapshot", 1).await; + prompt(app, &client, &session, "before snapshot", 1).await; let messages = client .get(&format!("/sessions/{session}/messages?order=asc")) .await @@ -110,7 +188,7 @@ impl Fixture { None => std::fs::remove_file(path).unwrap(), } } - prompt(&app, &client, &session, "after snapshot", 2).await; + prompt(app, &client, &session, "after snapshot", 2).await; let mut selection = json!({"message_id":first,"target":"code","action":"commit","expected_directory":project}); let preview = client .post( @@ -146,51 +224,13 @@ impl Fixture { for index in 0..2 { assert_eq!(journal.join(format!("stage_{index}")).exists(), displaced); } - let config = Arc::new(Mutex::new(json!({"snapshots":{"retention_days":7}}))); - let snapshots = maintenance( - &paths, - config.clone(), - cyber_app::snapshot_reclamation::policy(app.store.clone()), - ); Self { - _temp: temp, - app, client, - model, - paths, - project, session, receipt, journal, - config, - snapshots, } } - fn age(&self) { - let owner = self.receipt.origin.activity_id.clone(); - // Only the retention clock is aged. Receipt/native ownership bytes and - // original event sequence remain those of the actual committed restore. - self.app - .store - .transaction(move |db| { - db.execute( - "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", - rusqlite::params![8i64 * 86400000, owner], - )?; - Ok(()) - }) - .unwrap(); - } - async fn reclaim(&self) -> Result { - self.snapshots.reclaim(self.project.to_str().unwrap()).await - } - fn record(&self) -> Option { - let receipt = self.receipt.clone(); - self.app - .store - .read(move |db| records::lookup(db, &receipt)) - .unwrap() - } } fn maintenance( paths: &Paths, diff --git a/crates/cyber-app/tests/support/location_retirement.rs b/crates/cyber-app/tests/support/location_retirement.rs new file mode 100644 index 000000000..e1d5b927b --- /dev/null +++ b/crates/cyber-app/tests/support/location_retirement.rs @@ -0,0 +1,290 @@ +//! Multiple real Locations share one App and its serialized Store writer. +use super::*; + +#[derive(Clone, Copy)] +enum Route { + Missing, + Replacement, + PrivateLink, +} +struct Displaced { + retained: PathBuf, + owner: PathBuf, + route: Route, +} +impl Displaced { + fn paths(f: &Fixture, route: Route) -> Self { + let retained = f.project.with_file_name("retained-original"); + let owner = f.paths.tmp.join("later-location-owner"); + std::fs::create_dir(&owner).unwrap(); + std::fs::write(owner.join("owner-source"), b"later Location bytes").unwrap(); + Self { + retained, + owner, + route, + } + } + fn new(f: &Fixture, route: Route) -> Self { + let displaced = Self::paths(f, route); + std::fs::rename(&f.project, &displaced.retained).unwrap(); + match route { + Route::Missing => {} + Route::Replacement => { + std::fs::create_dir(&f.project).unwrap(); + std::fs::write( + f.project.join("owner-source"), + b"replacement Location bytes", + ) + .unwrap(); + } + Route::PrivateLink => std::os::unix::fs::symlink(&displaced.owner, &f.project).unwrap(), + } + // Later source ownership must survive both refusal and eventual cleanup. + std::fs::write(displaced.retained.join("file_0"), b"later source owner").unwrap(); + displaced + } + fn restore(self, f: &Fixture) { + assert_eq!( + std::fs::read(self.owner.join("owner-source")).unwrap(), + b"later Location bytes" + ); + match self.route { + Route::Missing => assert!(f.project.symlink_metadata().is_err()), + Route::Replacement => { + assert_eq!( + std::fs::read(f.project.join("owner-source")).unwrap(), + b"replacement Location bytes" + ); + std::fs::remove_file(f.project.join("owner-source")).unwrap(); + std::fs::remove_dir(&f.project).unwrap(); + } + Route::PrivateLink => { + assert_eq!(std::fs::read_link(&f.project).unwrap(), self.owner); + std::fs::remove_file(&f.project).unwrap(); + } + } + std::fs::rename(self.retained, &f.project).unwrap(); + } +} + +#[tokio::test] +async fn unavailable_old_location_defers_payload_but_not_other_location_retirement() { + exercise(false).await; +} +#[tokio::test] +async fn unavailable_old_location_preserves_pending_metadata_and_other_locations_progress() { + exercise(true).await; +} + +async fn exercise(metadata_pending: bool) { + for route in [Route::Missing, Route::Replacement, Route::PrivateLink] { + exercise_route(metadata_pending, route).await; + } +} +async fn exercise_route(metadata_pending: bool, route: Route) { + let f = Fixture::new().await; + let other = f.project.with_file_name("other-location"); + std::fs::create_dir(&other).unwrap(); + let restored = Restore::new(&f.app, &f.paths, &other, Some("after!\n".repeat(4096))).await; + f.age(); + age(&f.app, &restored.receipt); + if metadata_pending { + assert!(f.reclaim().await.unwrap() > 0); + native_retirement::plan_before_deletion(&f).await; + } + let witness = Witness::new(&f); + let displaced = Displaced::new(&f, route); + assert!(f.reclaim().await.is_err()); + if metadata_pending { + assert!( + f.snapshots + .retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await + .is_err() + ); + } + f.snapshots.gc().await; + witness.unchanged(&f); + assert_other_retired(&f, &restored, &other).await; + resume(&f, displaced, &witness).await; +} +async fn resume(f: &Fixture, displaced: Displaced, witness: &Witness) { + displaced.restore(f); + f.snapshots.gc().await; + assert!(f.record().unwrap().retired); + assert!(!f.journal.exists()); + assert_eq!( + std::fs::read(f.project.join("file_0")).unwrap(), + b"later source owner" + ); + let reply = f + .client + .post( + &format!( + "/sessions/{}/rewind/restores/{}/acknowledge", + f.session, f.receipt.origin.activity_id + ), + json!({}), + ) + .await + .unwrap(); + assert_eq!( + serde_json::from_value::(reply["data"].clone()).unwrap(), + f.receipt + ); + assert_eq!(sequence(&f.app, &f.session), witness.session_sequence); + assert_eq!( + sequence(&f.app, &f.receipt.origin.activity_id), + witness.owner_sequence + ); + assert_eq!(f.model.requests().len(), witness.requests); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn location_disappears_after_a_native_metadata_unlink_and_resumes_the_same_plan() { + let f = Fixture::new().await; + f.age(); + assert!(f.reclaim().await.unwrap() > 0); + let witness = Witness::new(&f); + let (snapshots, displaced) = relocate_after_intent(&f); + assert!( + snapshots + .retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await + .is_err() + ); + assert!(!f.project.exists()); + assert!(!f.journal.join("intent.json").exists()); + assert!(f.journal.join("intent.next").exists()); + let pending = f.record().unwrap(); + assert!(pending.completed && !pending.retired); + let plan = pending.metadata.unwrap(); + assert_eq!(sequence(&f.app, &records::Plan::id(&f.receipt).unwrap()), 2); + resume(&f, displaced, &witness).await; + assert_eq!(f.record().unwrap().metadata, Some(plan)); +} + +fn relocate_after_intent(f: &Fixture) -> (Arc, Displaced) { + let displaced = Displaced::paths(f, Route::Missing); + let retained = displaced.retained.clone(); + let source = f.project.clone(); + let intent = f.journal.join("intent.json"); + let snapshots = GitSnapshots::with_reclamation( + f.paths.data.clone(), + Arc::new(move |_| { + if !intent.exists() && source.exists() { + std::fs::rename(&source, &retained).unwrap(); + std::fs::write(retained.join("file_0"), b"later source owner").unwrap(); + } + json!({"snapshots":{"retention_days":7}}) + }), + vec![ + f.paths.config.clone(), + f.paths.state.clone(), + f.paths.cache.clone(), + f.paths.tmp.clone(), + ], + None, + Some(cyber_app::snapshot_reclamation::policy(f.app.store.clone())), + ); + (snapshots, displaced) +} + +struct Witness { + journal: std::collections::BTreeMap>, + workers: std::collections::BTreeMap>, + metadata: Option, + reclamation_sequence: i64, + session_sequence: i64, + owner_sequence: i64, + requests: usize, +} +impl Witness { + fn new(f: &Fixture) -> Self { + Self { + journal: image_bytes(&f.journal), + workers: image_bytes(&workers(f)), + metadata: f.record().and_then(|record| record.metadata), + reclamation_sequence: sequence(&f.app, &records::Plan::id(&f.receipt).unwrap()), + session_sequence: sequence(&f.app, &f.session), + owner_sequence: sequence(&f.app, &f.receipt.origin.activity_id), + requests: f.model.requests().len(), + } + } + fn unchanged(&self, f: &Fixture) { + assert_eq!(image_bytes(&f.journal), self.journal); + assert_eq!(image_bytes(&workers(f)), self.workers); + assert_eq!(f.record().and_then(|record| record.metadata), self.metadata); + assert_eq!( + sequence(&f.app, &records::Plan::id(&f.receipt).unwrap()), + self.reclamation_sequence + ); + assert_eq!(sequence(&f.app, &f.session), self.session_sequence); + assert_eq!( + sequence(&f.app, &f.receipt.origin.activity_id), + self.owner_sequence + ); + assert_eq!(f.model.requests().len(), self.requests); + } +} +fn workers(f: &Fixture) -> PathBuf { + f.journal + .parent() + .unwrap() + .parent() + .unwrap() + .join("workers") +} +fn sequence(app: &App, aggregate: &str) -> i64 { + let aggregate = aggregate.to_owned(); + app.store + .read(move |db| { + Ok(db.query_row( + "SELECT COALESCE(MAX(seq),-1) FROM event WHERE aggregate_id=?1", + [aggregate], + |row| row.get(0), + )?) + }) + .unwrap() +} +fn age(app: &App, receipt: &SnapshotRestoreReceipt) { + let owner = receipt.origin.activity_id.clone(); + app.store + .transaction(move |db| { + db.execute( + "UPDATE event SET time=time-?1 WHERE aggregate_id=?2", + rusqlite::params![8i64 * 86400000, owner], + )?; + Ok(()) + }) + .unwrap(); +} +async fn assert_other_retired(f: &Fixture, restored: &Restore, other: &std::path::Path) { + let policy = cyber_app::snapshot_reclamation::policy(f.app.store.clone()); + let record = policy + .record(&restored.receipt.clone().into()) + .unwrap() + .unwrap(); + assert!(record.completed && record.retired); + assert!(!restored.journal.exists()); + assert_eq!( + std::fs::read(other.join("file_0")).unwrap(), + "before\n".repeat(4096).as_bytes() + ); + let reply = restored + .client + .post( + &format!( + "/sessions/{}/rewind/restores/{}/acknowledge", + restored.session, restored.receipt.origin.activity_id + ), + json!({}), + ) + .await + .unwrap(); + assert_eq!( + serde_json::from_value::(reply["data"].clone()).unwrap(), + restored.receipt + ); +} diff --git a/crates/cyber-app/tests/support/snapshot_native_retirement.rs b/crates/cyber-app/tests/support/snapshot_native_retirement.rs index 41f36864c..57d31c4b8 100644 --- a/crates/cyber-app/tests/support/snapshot_native_retirement.rs +++ b/crates/cyber-app/tests/support/snapshot_native_retirement.rs @@ -262,6 +262,24 @@ impl Policy for Cut { } } +pub(super) async fn plan_before_deletion(f: &Fixture) { + let cut = maintenance( + &f.paths, + f.config.clone(), + Arc::new(Cut { + inner: cyber_app::snapshot_reclamation::policy(f.app.store.clone()), + after: false, + tamper: None, + }), + ); + let error = cut + .retire(f.project.to_str().unwrap(), f.receipt.clone().into()) + .await + .unwrap_err(); + assert!(error.contains("injected metadata interruption")); + assert!(f.record().unwrap().metadata.is_some()); +} + #[tokio::test] async fn native_retirement_resumes_before_and_after_deletion_without_rewriting_acknowledgement() { for after in [false, true] { diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index f05d65a83..5a2ea4bb2 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -59,6 +59,19 @@ code changes. The workflow also runs the four zero-payload CLI cases and retains their log. This resolves the inspected fixture routing problem in configuration; fresh-source native execution is still required to prove acceptance. +### Location continuity acceptance + +Location continuity acceptance now exercises six combinations of missing, +replacement or protected alias routes with payload cleanup or a committed metadata +plan. The same App/serialized Store retires another settled Location while preserving +the unavailable owner's exact journal/worker bytes and event sequences. Returning +the original directory resumes cleanup and data-only receipt acknowledgement. +A further race moves the Location after the actual active-intent unlink and resumes +the identical plan through Store discovery. Later source edits remain untouched. +Permanent-root removal still requires detached maintenance; the current live-root +checks remain in force. Failed reclamation App fixtures are retained, and their CI +step now runs independently and uploads them under the short fixture root. + ### Focused code-review findings and next actions This review covers incoming JSONL behavior and merge overlap, not acceptance of diff --git a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md index 554224baa..ace8dd9bd 100644 --- a/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md +++ b/openspec/changes/complete-daily-coding-surfaces/managed-recovery.md @@ -1019,6 +1019,19 @@ callback. The first writer frame supports empty complete checkout inventories; managed checkout claim retirement, partial constructors and all actual native retirement/reconciliation/crash paths remain required before M6 acceptance. +#### Location route continuity + +The existing retirement route requires a live original Location identity. Missing, +replacement and private-root alias routes defer remaining cleanup and preserve +their committed metadata plan. A separate fully bound Location remains eligible +through the same App/Store maintenance pass. Returning the original directory +permits the same frozen plan to resume, including after the active intent has +already been unlinked; retained Store discovery must find that owner. Source edits +made while the root was displaced remain untouched, and receipt acknowledgement +after retirement remains data-only. Permanently removed roots still require a +separately validated detached maintenance path; these route safeguards do not +implement that capability. + #### Consumed managed checkout retirement The managed extension SHALL freeze the complete ordered checkout bindings for each diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md index fd4912aa5..d385a4def 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/snapshots-checkpoints/spec.md @@ -534,3 +534,12 @@ - **AND** foreign or duplicate worker roles, supplied paths, changed plan identities, missing payload completion and inconsistent byte counts SHALL refuse - **AND** later native-owner fences and incomplete reference views SHALL refuse new retirement writes; historical projection rebuilds SHALL preserve immutable history without authorizing native I/O - **AND** accepting the event schema alone SHALL NOT prove native metadata deletion or relax independent checkout, namespace, lifetime, reader or retention checks + +#### Scenario: Root-bound cleanup loses the original Location route +- **GIVEN** native payload or metadata cleanup requires the original live Location identity +- **WHEN** that route disappears, resolves to a replacement directory or redirects into protected private state +- **THEN** the root-bound path SHALL preserve the remaining private payloads, worker metadata and committed retirement plan without new completion or owner settlement +- **AND** a separate settled Location with complete ownership and reference evidence SHALL remain eligible for independent maintenance +- **WHEN** the exact original directory returns +- **THEN** periodic maintenance SHALL resume its existing plan without replaying source effects, changing later source edits, moving the transcript or starting inference +- **AND** this route SHALL revalidate Location identity after each metadata unlink before consuming another leaf diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 51a79004a..2b160f1db 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1422,3 +1422,56 @@ cleanup. The two file-copy zero-payload cases also pass in the default build in branch. Full workspace/SDK runtime and current-source platform acceptance remain required. Constructor, old/removed Location and remaining native cut/client acceptance stay open with M6/M8 and all parent tasks. + +Location-route acceptance continuation: the existing producer still requires the +original live directory identity. New tests use two actual Locations and Sessions +in one App/serialized Store to cover missing, replacement and private-root alias +routes, both before payload reclamation and with a committed native metadata plan. +The unavailable owner retains exact journal/worker bytes, plan and event sequences; +the other Location completes payload and metadata retirement during that same GC +pass. Returning the original directory resumes cleanup without changing later +source edits, Session/native-owner sequences or inference count, and explicit HTTP +acknowledgement returns the retained receipt. This is six route/phase combinations. + +A seventh case uses the actual configuration callback after native active-intent +unlink to rename the source directory. Native verification refuses before removing +the spare or worker metadata, leaving sequence 2 and the exact stored plan. After +the directory returns, Store discovery finds the owner without its intent and +periodic maintenance completes that identical plan. No native delete or source +effect hook is replaced. The initial race assertion mistakenly expected spare-first +ordering; observation now follows the producer's actual intent-first order. Its +failed partial-retirement fixture was preserved and inspected. + +Shared real-restore setup is extracted into a reusable `Restore` fixture so the +second Location uses the same App and writer rather than a fabricated receipt or +second Store. Reclamation App fixtures now preserve their root on panic. CI runs +this App suite independently of earlier failures under the same short retained +fixture root as Git/CLI proofs; its source filter explicitly includes the new +support module. No production API, wire schema, generated SDK or default Git +admission changes. + +The full default reclamation selection before adding the final race passes 44 +entries in 289.68 seconds. The final three Location tests, including all seven +scenarios and native-owner sequence assertions, pass in 36.44 seconds by default +and 37.03 seconds with native-git-acceptance under a short fixture root. Default +and native-feature App all-target Clippy with warnings denied pass after separating orchestration +and resumption (the initial combined helper measured 24/15; no allowance added). +Focused code-review-skill review covers original-identity refusal, unrelated +Location progress, pending-plan preservation, post-unlink route revalidation, +exact-plan resumption, source/event invariants and failure evidence retention; +no outstanding finding is identified in this test continuation. Whole-branch +review, full workspace/SDK runtime, managed removed-Location cases, constructor +cuts and fresh-source platform acceptance remain required. + +Formatting, whitespace, workflow YAML and independent-step/retained-root checks +pass. Strict OpenSpec validates all 100 items; spec lint reports 0 errors and 25 +warnings. Native C deprecation and linker unwind diagnostics were observed without +assigning origin. Fetching main again finds no upstream commits missing from the +branch. The native workflow must execute the new source before accepting these +cases as cross-platform evidence. + +These tests verify the current live-root route. Positive maintenance after +permanent root removal still needs a separately validated detached authority and +coordination path; the current checks were not relaxed or replaced, and this +remaining capability has not been marked complete. M6/M8, parent tasks and issue +#93 remain open. From 70e9495bbbbdd622facfac124b6cafef055d7085 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 07:05:39 -0400 Subject: [PATCH 72/87] Bind direct JSONL requests to recorded input ownership --- .github/workflows/non-git-snapshot-proof.yml | 12 +- crates/cyber-cli/src/commands/exec/driver.rs | 59 +++-- .../src/commands/exec/driver/request_tests.rs | 225 ++++++++++++++++ crates/cyber-cli/src/commands/exec/inputs.rs | 42 +++ crates/cyber-cli/tests/exec_stream_input.rs | 23 ++ crates/cyber-server/src/runtime/drain.rs | 1 + crates/cyber-server/src/runtime/events.rs | 3 + crates/cyber-server/src/runtime/mod.rs | 8 + crates/cyber-server/src/runtime/model.rs | 13 +- crates/cyber-server/src/runtime/requests.rs | 25 ++ .../src/runtime/requests/input_owners.rs | 242 ++++++++++++++++++ crates/cyber-server/tests/requests.rs | 162 ++++++++++++ docs/goal-delivery.md | 17 +- docs/local-core-guide.md | 5 +- .../complete-daily-coding-surfaces/design.md | 15 ++ .../specs/exec-mode/spec.md | 14 + .../complete-daily-coding-surfaces/tasks.md | 51 +++- sdk/openapi.json | 7 + sdk/typescript/src/generated/types.ts | 5 + 19 files changed, 897 insertions(+), 32 deletions(-) create mode 100644 crates/cyber-cli/src/commands/exec/driver/request_tests.rs create mode 100644 crates/cyber-server/src/runtime/requests/input_owners.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 4b7f5d0b5..32291dd25 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -15,6 +15,11 @@ on: - crates/cyber-server/src/runtime/drain.rs - crates/cyber-server/src/runtime/structured*.rs - crates/cyber-server/src/runtime/model.rs + - crates/cyber-server/src/runtime/mod.rs + - crates/cyber-server/src/runtime/events.rs + - crates/cyber-server/src/runtime/requests.rs + - crates/cyber-server/src/runtime/requests/** + - crates/cyber-server/tests/requests.rs - crates/cyber-server/tests/structured_root.rs - crates/cyber-server/tests/support/structured_terminal_fault.rs - crates/cyber-server/src/http/sessions.rs @@ -153,10 +158,15 @@ jobs: cargo test --locked -p cyber-cli --features native-git-acceptance --test branches_cli zero_payload:: -- --nocapture --test-threads=1 2>&1 | tee zero-payload-retirement-proof.log - name: JSONL driver and atomic last-message regressions if: ${{ !cancelled() }} + env: + TMPDIR: /tmp/c${{ github.run_number }} shell: bash run: | set -euo pipefail - cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::exec:: -- --nocapture 2>&1 | tee jsonl-driver-proof.log + mkdir -p "$TMPDIR" + cargo test --locked -p cyber-server --lib runtime::requests::input_owners:: -- --nocapture 2>&1 | tee jsonl-driver-proof.log + cargo test --locked -p cyber-server --test requests -- --nocapture 2>&1 | tee -a jsonl-driver-proof.log + cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::exec:: -- --nocapture 2>&1 | tee -a jsonl-driver-proof.log cargo test --locked -p cyber-cli --features native-git-acceptance --test exec_stream_input -- --nocapture 2>&1 | tee -a jsonl-driver-proof.log - name: Root structured result recovery and CLI stop regressions if: ${{ !cancelled() }} diff --git a/crates/cyber-cli/src/commands/exec/driver.rs b/crates/cyber-cli/src/commands/exec/driver.rs index 593a9b807..fdbf04d86 100644 --- a/crates/cyber-cli/src/commands/exec/driver.rs +++ b/crates/cyber-cli/src/commands/exec/driver.rs @@ -38,7 +38,7 @@ enum Line { Eof, } -/// A permission or question request this Session raised, with the Session to reply to. +/// A live permission/question request bound to this driver's exact admitted inputs. struct Ask { session_id: String, question: bool, @@ -90,24 +90,7 @@ pub async fn execute( let owned = Owned::capture(client, &id).await?; let (sender, mut lines) = mpsc::channel(64); tokio::spawn(read_stdin(sender)); - let mut driver = Driver { - client, - id: &id, - args, - out: &mut out, - run, - owned, - inputs: Inputs::default(), - answering: Vec::new(), - answer_done: false, - asks: BTreeMap::new(), - closed: false, - idle: true, - fatal: None, - interrupted: false, - last_text: String::new(), - lease: None, - }; + let mut driver = Driver::new(client, &id, args, &mut out, run, owned); driver.attach().await; driver.drive(&mut events, &mut lines).await; driver.release().await; @@ -143,7 +126,35 @@ async fn read_stdin(sender: mpsc::Sender) { } } -impl Driver<'_> { +impl<'a> Driver<'a> { + fn new( + client: &'a Client, + id: &'a str, + args: &'a ExecArgs, + out: &'a mut Out, + run: Run, + owned: Owned, + ) -> Self { + Self { + client, + id, + args, + out, + run, + owned, + inputs: Inputs::default(), + answering: Vec::new(), + answer_done: false, + asks: BTreeMap::new(), + closed: false, + idle: true, + fatal: None, + interrupted: false, + last_text: String::new(), + lease: None, + } + } + async fn drive + Unpin>( &mut self, events: &mut S, @@ -261,7 +272,7 @@ impl Driver<'_> { } } - /// Answer a request this Session raised and is still waiting on; any other ID is a + /// Answer a request bound to this driver's live inputs; any other ID is a /// protocol error that leaves the run going. async fn reply(&mut self, request_id: &str, question: bool, body: Value) { let owned = self @@ -447,6 +458,9 @@ impl Driver<'_> { } async fn asked(&mut self, base: &str, data: &Value) { + if !self.inputs.owns_request(self.id, data) { + return; + } let (Some(request_id), Some(session_id)) = (data["id"].as_str(), data["session_id"].as_str()) else { @@ -575,3 +589,6 @@ impl Driver<'_> { code } } + +#[cfg(test)] +mod request_tests; diff --git a/crates/cyber-cli/src/commands/exec/driver/request_tests.rs b/crates/cyber-cli/src/commands/exec/driver/request_tests.rs new file mode 100644 index 000000000..bae1f303b --- /dev/null +++ b/crates/cyber-cli/src/commands/exec/driver/request_tests.rs @@ -0,0 +1,225 @@ +//! Two driver controllers, one actual App/Session and real model/tool requests. +#[path = "../../../../../cyber-app/tests/support/scripted_model.rs"] +mod scripted_model; +use super::*; +use crate::cli::Format; +use cyber_app::{App, AppOptions}; +use cyber_core::paths::{DatabaseLocation, Paths}; +use cyber_server::runtime::{PendingKind, PendingRequest}; + +struct Fixture { + temp: tempfile::TempDir, + app: App, + client: Client, + session: String, + _model: scripted_model::ScriptedModel, +} +impl Drop for Fixture { + fn drop(&mut self) { + if std::thread::panicking() { + self.temp.disable_cleanup(true); + eprintln!( + "Preserved shared-driver fixture at {}", + self.temp.path().display() + ); + } + } +} +impl Fixture { + async fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let paths = Paths { + data: root.join("data"), + config: root.join("config"), + state: root.join("state"), + cache: root.join("cache"), + tmp: root.join("tmp"), + }; + paths.ensure().unwrap(); + let project = root.join("project"); + std::fs::create_dir(&project).unwrap(); + let model = scripted_model::ScriptedModel::start(response); + let mut config = model.config(); + config["permissions"] = json!({"question":"allow","bash":"ask"}); + std::fs::write(paths.config.join("cyber.jsonc"), config.to_string()).unwrap(); + let app = App::build(AppOptions { + paths: paths.clone(), + home: root.join("home"), + database: DatabaseLocation::File(paths.data.join("test.db")), + default_directory: project.clone(), + sandbox_policy: None, + snapshots: false, + interactive: false, + password: None, + mcp_credentials: None, + remote_skill_fetcher: None, + workflow_cores: None, + isolation: None, + }) + .await + .unwrap(); + let client = Client::embedded(app.embedded()).at(project.to_str().unwrap()); + let session = client + .post( + "/sessions", + json!({"model":"test/main","mode":"default","unattended":true}), + ) + .await + .unwrap()["data"]["id"] + .as_str() + .unwrap() + .to_owned(); + Self { + temp, + app, + client, + session, + _model: model, + } + } + async fn request(&self) -> PendingRequest { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if let Some(request) = self + .app + .runtime + .pending_requests(Some(&self.session)) + .into_iter() + .next() + { + return request; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("model/tool request did not appear") + } +} +fn response(body: &Value) -> Value { + if !body["tools"].is_array() + || body["messages"] + .as_array() + .and_then(|messages| messages.last()) + .is_some_and(|last| last["role"] == "tool") + { + return json!({"content":"done"}); + } + let question = body.to_string().contains("QUESTION_CASE"); + let (name, arguments) = if question { + ( + "question", + json!({"questions":[{"question":"Choose","header":"Choice","options":[{"label":"yes"},{"label":"no"}]}]}), + ) + } else { + ("bash", json!({"command":"echo must-not-run"})) + }; + let id = format!("call_{}", body["messages"].as_array().unwrap().len()); + json!({"tool_calls":[{"index":0,"id":id,"type":"function","function":{"name":name,"arguments":arguments.to_string()}}]}) +} +fn event(request: &PendingRequest) -> (&'static str, Value) { + let base = if matches!(request.kind, PendingKind::Question { .. }) { + "question.asked" + } else { + "permission.asked" + }; + (base, serde_json::to_value(request).unwrap()) +} +fn reply(question: bool) -> Value { + if question { + json!({"answers":[["yes"]]}) + } else { + json!({"reply":"reject","message":"owner declines"}) + } +} + +#[tokio::test] +async fn shared_session_requests_replies_and_close_keep_exact_driver_ownership() { + for text in ["PERMISSION_CASE", "QUESTION_CASE"] { + exercise(text).await; + } +} +async fn exercise(text: &str) { + let f = Fixture::new().await; + let args = ExecArgs::default(); + let session = json!({"id":f.session}); + let mut first_out = Out::new(Format::Text, &args, &session); + let mut second_out = Out::new(Format::Text, &args, &session); + let mut first = Driver::new( + &f.client, + &f.session, + &args, + &mut first_out, + Run::new(&args, Instant::now()), + Owned::capture(&f.client, &f.session).await.unwrap(), + ); + let mut second = Driver::new( + &f.client, + &f.session, + &args, + &mut second_out, + Run::new(&args, Instant::now()), + Owned::capture(&f.client, &f.session).await.unwrap(), + ); + first.attach().await; + second.attach().await; + first + .admit( + Some("same-correlation-label".into()), + vec![json!({"type":"text","text":text})], + "queue", + ) + .await; + let request = f.request().await; + let question = matches!(request.kind, PendingKind::Question { .. }); + assert_eq!(question, text == "QUESTION_CASE"); + second + .admit( + Some("same-correlation-label".into()), + vec![json!({"type":"text","text":text})], + "steer", + ) + .await; + protect_first(&f, &mut first, &mut second, &request).await; + finish_second(&f, &mut first, &mut second).await; + first.close().await; + f.app.runtime.shutdown().await; +} +async fn protect_first( + f: &Fixture, + first: &mut Driver<'_>, + second: &mut Driver<'_>, + request: &PendingRequest, +) { + let question = matches!(request.kind, PendingKind::Question { .. }); + let (base, data) = event(request); + second.asked(base, &data).await; + assert!(second.asks.is_empty()); + second.reply(&request.id, question, reply(question)).await; + assert_eq!( + f.app.runtime.pending_requests(Some(&f.session))[0].id, + request.id + ); + second.close().await; + assert_eq!( + f.app.runtime.pending_requests(Some(&f.session))[0].id, + request.id + ); + first.asked(base, &data).await; + assert_eq!(first.asks.len(), 1); + first.reply(&request.id, question, reply(question)).await; +} +async fn finish_second(f: &Fixture, first: &mut Driver<'_>, second: &mut Driver<'_>) { + let request = f.request().await; + let (base, data) = event(&request); + first.asked(base, &data).await; + assert!(first.asks.is_empty()); + second.asked(base, &data).await; + // The closed driver declines only its own late request. + assert!(second.asks.is_empty()); + tokio::time::timeout(Duration::from_secs(30), f.app.runtime.wait_idle(&f.session)) + .await + .unwrap(); + assert!(f.app.runtime.pending_requests(Some(&f.session)).is_empty()); +} diff --git a/crates/cyber-cli/src/commands/exec/inputs.rs b/crates/cyber-cli/src/commands/exec/inputs.rs index 91df95c87..40ae05b0f 100644 --- a/crates/cyber-cli/src/commands/exec/inputs.rs +++ b/crates/cyber-cli/src/commands/exec/inputs.rs @@ -60,6 +60,25 @@ impl Inputs { self.pending.contains_key(message_id) } + /// Only a complete server-recorded cohort can authorize a driver reply. + /// Current Session activity and assistant history are not ownership proof. + pub fn owns_request(&self, session: &str, data: &Value) -> bool { + if data["session_id"].as_str() != Some(session) + || data["message_id"].as_str().is_none_or(str::is_empty) + || data["call_id"].as_str().is_none_or(str::is_empty) + { + return false; + } + let Some(inputs) = data["input_message_ids"].as_array() else { + return false; + }; + !inputs.is_empty() + && inputs.len() <= MAX_PENDING + && inputs + .iter() + .all(|input| input.as_str().is_some_and(|id| self.contains(id))) + } + pub fn get_mut(&mut self, message_id: &str) -> Option<&mut Input> { self.pending.get_mut(message_id) } @@ -118,6 +137,29 @@ mod tests { input } + #[test] + fn request_ownership_requires_every_live_input_and_exact_session_call_evidence() { + let mut inputs = Inputs::default(); + inputs.admit(input(0)).unwrap(); + inputs.admit(input(1)).unwrap(); + let owned = serde_json::json!({"session_id":"session","message_id":"assistant","call_id":"call","input_message_ids":["msg_0","msg_1"]}); + assert!(inputs.owns_request("session", &owned)); + for (field, value) in [ + ("session_id", serde_json::json!("other")), + ("message_id", Value::Null), + ("call_id", Value::Null), + ("input_message_ids", serde_json::json!([])), + ("input_message_ids", serde_json::json!(["msg_0", "foreign"])), + ("input_message_ids", serde_json::json!(["msg_0", 3])), + ] { + let mut unknown = owned.clone(); + unknown[field] = value; + assert!(!inputs.owns_request("session", &unknown), "{unknown}"); + } + inputs.complete("msg_0").unwrap(); + assert!(!inputs.owns_request("session", &owned)); + } + #[test] fn completed_payloads_release_capacity_and_failures_survive_long_runs() { let mut inputs = Inputs::default(); diff --git a/crates/cyber-cli/tests/exec_stream_input.rs b/crates/cyber-cli/tests/exec_stream_input.rs index 59c18b03e..602b6b73d 100644 --- a/crates/cyber-cli/tests/exec_stream_input.rs +++ b/crates/cyber-cli/tests/exec_stream_input.rs @@ -459,7 +459,9 @@ fn a_permission_is_answered_over_stdin_and_others_are_refused() { let mut run = Driven::start(&provider, &[]); let mut seen = Vec::new(); run.send(r#"{"type":"user","text":"run it"}"#); + let input = run.until("input", &mut seen); let asked = run.until("permission_request", &mut seen); + assert_eq!(asked["input_message_ids"], json!([input["message_id"]])); let request = asked["id"].as_str().unwrap().to_owned(); // A reply to a request this run does not have is an error, and the run goes on. run.send(r#"{"type":"permission_reply","request_id":"per_unknown","reply":"once"}"#); @@ -492,6 +494,27 @@ fn eof_while_a_request_waits_declines_it_instead_of_hanging() { assert_eq!(run.exit(), Some(0)); } +#[test] +fn a_question_is_bound_to_its_input_and_answered_over_stdin() { + let provider = Provider::start(&[ + r#"question {"questions":[{"question":"Choose","header":"Choice","options":[{"label":"yes"},{"label":"no"}]}]}"#, + "text:choice received", + ]); + let mut run = Driven::start(&provider, &[]); + let mut seen = Vec::new(); + run.send(r#"{"type":"user","text":"choose","id":"choice"}"#); + let input = run.until("input", &mut seen); + let asked = run.until("question", &mut seen); + assert_eq!(asked["input_message_ids"], json!([input["message_id"]])); + run.send( + &json!({"type":"question_reply","request_id":asked["id"],"answers":[["yes"]]}).to_string(), + ); + let result = run.until("result", &mut seen); + assert_eq!(result["id"], "choice"); + assert_eq!(result["result"], "choice received"); + assert_eq!(run.exit(), Some(0)); +} + #[test] fn a_malformed_line_is_a_protocol_error_exit_2() { let provider = Provider::start(&[]); diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 9f50c60ff..999578179 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -1260,6 +1260,7 @@ impl Inner { let message_id = cyber_core::ids::new_id("msg"); let origin = super::usage_origin::step_origin(&self.store, &state)?; let payload = StepStarted { + input_message_ids: super::requests::input_owners::capture(&state), agent: Some(state.info.agent.clone()), mode: Some(mode), message_id: message_id.clone(), diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 676df84df..3cc3ce1b9 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -265,6 +265,9 @@ pub struct SystemAdded { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StepStarted { + /// Complete active input cohort captured before this inference step. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub input_message_ids: Vec, /// Agent identity pinned for inference and every tool group of this Turn. #[serde(default, skip_serializing_if = "Option::is_none")] pub agent: Option, diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 14df9c65d..519dd0e8a 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -2768,6 +2768,14 @@ impl Inner { if events.is_empty() { return Ok(Vec::new()); } + for event in &events { + if event.kind == events::STEP_STARTED { + let step: events::StepStarted = serde_json::from_value(event.data.clone()) + .map_err(|error| RuntimeError::Corrupt(error.to_string()))?; + requests::input_owners::validate(state, &step.input_message_ids) + .map_err(RuntimeError::Invalid)?; + } + } let stored = self .store .append(&state.info.id, Expected::Seq(state.last_seq), events)?; diff --git a/crates/cyber-server/src/runtime/model.rs b/crates/cyber-server/src/runtime/model.rs index 83f8f6196..93d02b6ab 100644 --- a/crates/cyber-server/src/runtime/model.rs +++ b/crates/cyber-server/src/runtime/model.rs @@ -448,6 +448,11 @@ pub struct LastStep { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] pub struct StepSnapshot { pub step_id: String, + /// Internal immutable provenance, reconstructed from step events on replay. + #[serde(skip)] + pub(crate) input_message_ids: Vec, + #[serde(skip)] + pub(crate) input_root: Option, /// The user message the step answers. pub user_message_id: Option, pub pre: Option, @@ -998,7 +1003,7 @@ impl SessionState { self.session_started = true; self.hook_stop_step = marker.stop_after_step; } - "session.step.started" => self.on_step_started(decode(e)?, e.time_ms), + "session.step.started" => self.on_step_started(decode(e)?, e.time_ms)?, "session.text.ended" | "session.reasoning.ended" => self.on_content(kind, decode(e)?), "session.tool.called" => self.on_called(decode(e)?), "session.tool.dispatched" => self.on_dispatched(decode(e)?), @@ -1414,7 +1419,8 @@ impl SessionState { }); } - fn on_step_started(&mut self, s: StepStarted, time_ms: i64) { + fn on_step_started(&mut self, s: StepStarted, time_ms: i64) -> Result<(), String> { + super::requests::input_owners::validate(self, &s.input_message_ids)?; self.first_turn_started = true; self.turn_mode = Some(s.mode.unwrap_or_else(|| self.info.mode.clone())); let agent = s.agent.unwrap_or_else(|| self.info.agent.clone()); @@ -1423,6 +1429,8 @@ impl SessionState { self.step_origin = s.origin; self.steps.push(StepSnapshot { step_id: s.message_id.clone(), + input_message_ids: s.input_message_ids, + input_root: super::requests::input_owners::current(self).map(|(_, id)| id.to_owned()), user_message_id: self.current_user_message().map(str::to_string), pre: s.snapshot.clone(), post: None, @@ -1435,6 +1443,7 @@ impl SessionState { started_ms: Some(time_ms), ..AssistantEntry::default() })); + Ok(()) } fn assistant(&mut self, id: &str) -> Option<&mut AssistantEntry> { diff --git a/crates/cyber-server/src/runtime/requests.rs b/crates/cyber-server/src/runtime/requests.rs index 3d335ae64..9a042ab82 100644 --- a/crates/cyber-server/src/runtime/requests.rs +++ b/crates/cyber-server/src/runtime/requests.rs @@ -4,6 +4,7 @@ //! A tool asks through its [`Asker`]; the request is recorded durably and waits until an //! attached client replies. Without an interactive client the reply is `Unattended` at once, //! so non-interactive Sessions never block. +pub(super) mod input_owners; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, PoisonError, Weak}; @@ -98,6 +99,10 @@ pub struct PendingRequest { pub session_id: String, pub call_id: String, pub message_id: String, + /// Complete input cohort driving this exact recorded step/call. Empty means + /// unknown; clients must not infer ownership from current Session state. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub input_message_ids: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub origin: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -325,6 +330,7 @@ impl Inner { session_id: asker.session_id.clone(), call_id: asker.call_id.clone(), message_id: asker.message_id.clone(), + input_message_ids: Vec::new(), origin: None, routed_to: Vec::new(), kind, @@ -363,6 +369,25 @@ impl Inner { QUESTION_ASKED }; let mut state = handle.state.lock().await; + let (session, message, call) = ( + asker.session_id.clone(), + asker.message_id.clone(), + asker.call_id.clone(), + ); + let owners = self + .store + .read(move |db| input_owners::recorded(db, &session, &message, &call)); + match owners { + Ok(owners) => request.input_message_ids = owners, + Err(error) => { + self.bus.publish(super::LiveEvent::Error { + session_id: asker.session_id.clone(), + kind: "storage".into(), + message: format!("could not establish request input provenance: {error}"), + }); + return false; + } + } let mut asked = event(kind_name, &request); if kind_name == PERMISSION_ASKED && super::workflow_solve::owns_verification(&asker.session_id) diff --git a/crates/cyber-server/src/runtime/requests/input_owners.rs b/crates/cyber-server/src/runtime/requests/input_owners.rs new file mode 100644 index 000000000..7d9678532 --- /dev/null +++ b/crates/cyber-server/src/runtime/requests/input_owners.rs @@ -0,0 +1,242 @@ +//! Capture branch-aware input provenance before inference; never infer it later. +use super::super::{InputStatus, SessionState}; +use cyber_store::StoreError; +use rusqlite::{Connection, OptionalExtension, params}; + +const MAX_INPUTS: usize = 32; +const MAX_ID_BYTES: usize = 256; +const MAX_ENCODED: usize = MAX_INPUTS * (MAX_ID_BYTES * 6 + 3) + 2; + +fn active_seq(state: &SessionState, id: &str) -> Option { + state + .input(id) + .filter(|row| row.status == InputStatus::Promoted && !row.forwarded_skill) + .and_then(|row| row.promoted_seq) +} +fn bounded(inputs: &[String]) -> bool { + !inputs.is_empty() + && inputs.len() <= MAX_INPUTS + && inputs + .iter() + .all(|id| !id.is_empty() && id.len() <= MAX_ID_BYTES) +} +pub(crate) fn validate(state: &SessionState, owners: &[String]) -> Result<(), String> { + if !owners.is_empty() && owners != capture(state) { + return Err("Step input provenance differs from its active path".into()); + } + Ok(()) +} +/// Active-path rows and step snapshots survive branch changes and compaction. +/// A legacy continuation without recorded provenance remains unknown. +pub(crate) fn current(state: &SessionState) -> Option<(i64, &str)> { + state + .inbox + .iter() + .filter(|row| row.status == InputStatus::Promoted && !row.forwarded_skill) + .filter_map(|row| row.promoted_seq.map(|seq| (seq, row.message_id.as_str()))) + .max_by_key(|(seq, _)| *seq) +} +pub(crate) fn capture(state: &SessionState) -> Vec { + let Some((seq, current)) = current(state) else { + return Vec::new(); + }; + let previous = state.steps.last(); + let floor = match previous.and_then(|step| step.input_root.as_deref()) { + Some(id) if id == current => { + let owners = previous + .map(|step| step.input_message_ids.as_slice()) + .unwrap_or_default(); + return if bounded(owners) && owners.iter().all(|id| active_seq(state, id).is_some()) { + owners.to_vec() + } else { + Vec::new() + }; + } + Some(id) => match active_seq(state, id) { + Some(value) => value, + None => return Vec::new(), + }, + None => -1, + }; + collect(state, floor, seq) +} +fn collect(state: &SessionState, floor: i64, through: i64) -> Vec { + let mut owners: Vec<_> = state + .inbox + .iter() + .filter(|row| !row.forwarded_skill && row.status == InputStatus::Promoted) + .filter_map(|row| row.promoted_seq.map(|seq| (seq, &row.message_id))) + .filter(|(seq, _)| *seq > floor && *seq <= through) + .take(MAX_INPUTS + 1) + .collect(); + if owners.len() > MAX_INPUTS { + return Vec::new(); + } + owners.sort_by_key(|(seq, _)| *seq); + let owners: Vec<_> = owners.into_iter().map(|(_, id)| id.clone()).collect(); + if bounded(&owners) { owners } else { Vec::new() } +} + +pub(super) fn recorded( + db: &Connection, + session: &str, + message: &str, + call: &str, +) -> Result, StoreError> { + let raw: Option> = db + .query_row( + "SELECT CASE WHEN json_type(s.data,'$.input_message_ids')='array' + AND json_array_length(s.data,'$.input_message_ids')<=?4 + AND length(CAST(json_extract(s.data,'$.input_message_ids') AS BLOB))<=?5 + THEN json_extract(s.data,'$.input_message_ids') ELSE NULL END + FROM event s WHERE s.aggregate_id=?1 AND s.type='session.step.started.1' + AND json_extract(s.data,'$.message_id')=?2 + AND EXISTS (SELECT 1 FROM event c WHERE c.aggregate_id=s.aggregate_id + AND c.type='session.tool.called.1' AND c.seq>s.seq + AND json_extract(c.data,'$.message_id')=?2 AND json_extract(c.data,'$.call_id')=?3) + ORDER BY s.seq DESC LIMIT 1", + params![ + session, + message, + call, + MAX_INPUTS as i64, + MAX_ENCODED as i64 + ], + |row| row.get(0), + ) + .optional()?; + let Some(raw) = raw.flatten() else { + return Ok(Vec::new()); + }; + let owners: Vec = serde_json::from_str(&raw) + .map_err(|_| StoreError::Unavailable("request input provenance is malformed".into()))?; + Ok(if bounded(&owners) { owners } else { Vec::new() }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn database() -> Connection { + let db = Connection::open_in_memory().unwrap(); + db.execute_batch("CREATE TABLE event(aggregate_id TEXT,seq INTEGER,type TEXT,data TEXT)") + .unwrap(); + db + } + fn insert(db: &Connection, seq: i64, kind: &str, data: serde_json::Value) { + db.execute( + "INSERT INTO event VALUES('session',?1,?2,?3)", + params![seq, kind, data.to_string()], + ) + .unwrap(); + } + fn promote(db: &Connection, seq: i64, id: &str) { + insert( + db, + seq, + "session.prompt.promoted.1", + json!({"message_id":id}), + ); + } + fn step(db: &Connection, seq: i64, id: &str, call: &str, owners: serde_json::Value) { + insert( + db, + seq, + "session.step.started.1", + json!({"message_id":id,"input_message_ids":owners}), + ); + insert( + db, + seq + 1, + "session.tool.called.1", + json!({"message_id":id,"call_id":call}), + ); + } + #[test] + fn exact_call_keeps_its_cohort_across_continuations_and_later_steering() { + let db = database(); + promote(&db, 1, "first"); + promote(&db, 2, "second"); + step(&db, 3, "assistant_a", "call_a", json!(["first", "second"])); + step(&db, 5, "assistant_b", "call_b", json!(["first", "second"])); + promote(&db, 7, "foreign"); + step(&db, 8, "assistant_c", "call_c", json!(["foreign"])); + for (message, call) in [("assistant_a", "call_a"), ("assistant_b", "call_b")] { + assert_eq!( + recorded(&db, "session", message, call).unwrap(), + ["first", "second"] + ); + } + assert_eq!( + recorded(&db, "session", "assistant_c", "call_c").unwrap(), + ["foreign"] + ); + for (session, message, call) in [ + ("other", "assistant_a", "call_a"), + ("session", "assistant_a", "call_b"), + ("session", "first", "call_a"), + ] { + assert!(recorded(&db, session, message, call).unwrap().is_empty()); + } + } + #[test] + fn excessive_or_unbound_evidence_never_becomes_partial_ownership() { + for count in [0, MAX_INPUTS, MAX_INPUTS + 1] { + let db = database(); + for index in 0..count { + promote(&db, index as i64, &format!("input_{index}")); + } + step( + &db, + 100, + "assistant", + "call", + json!( + (0..count) + .map(|index| format!("input_{index}")) + .collect::>() + ), + ); + let owners = recorded(&db, "session", "assistant", "call").unwrap(); + assert_eq!( + owners.len(), + if count == MAX_INPUTS { MAX_INPUTS } else { 0 } + ); + } + let db = database(); + promote(&db, 1, &"x".repeat(MAX_ID_BYTES + 1)); + step( + &db, + 2, + "assistant", + "call", + json!(["x".repeat(MAX_ID_BYTES + 1)]), + ); + assert!( + recorded(&db, "session", "assistant", "call") + .unwrap() + .is_empty() + ); + } + #[test] + fn malformed_cohort_is_an_error_and_legacy_requests_remain_unknown() { + let db = database(); + insert( + &db, + 1, + "session.prompt.promoted.1", + json!({"foreign":"unknown"}), + ); + step(&db, 2, "assistant", "call", json!([true])); + assert!(recorded(&db, "session", "assistant", "call").is_err()); + let request: super::super::PendingRequest=serde_json::from_value(json!({"id":"per_old","session_id":"session","message_id":"assistant","call_id":"call","kind":"permission","action":"bash","resources":[],"always_patterns":[]})).unwrap(); + assert!(request.input_message_ids.is_empty()); + assert!( + serde_json::to_value(request) + .unwrap() + .get("input_message_ids") + .is_none() + ); + } +} diff --git a/crates/cyber-server/tests/requests.rs b/crates/cyber-server/tests/requests.rs index 107e17b50..dcadfee4c 100644 --- a/crates/cyber-server/tests/requests.rs +++ b/crates/cyber-server/tests/requests.rs @@ -80,6 +80,168 @@ async fn once_lets_the_tool_run_and_is_recorded() { ); } +#[tokio::test] +async fn requests_bind_the_recorded_input_cohort_instead_of_later_session_input() { + let h = Harness::new(Setup { + compaction: cyber_server::runtime::CompactionConfig { + auto: false, + keep_tokens: 0, + keep_turns: Some(0), + ..Default::default() + }, + scripts: vec![ + ( + "test/main", + vec![ + tools(&[("c0", "shell", "{}")]), + tools(&[("c1", "shell", "{}")]), + tools(&[("c2", "ask", "{}")]), + text("done"), + ], + ), + ("test/summary", vec![text("summary")]), + ], + ..Setup::default() + }); + h.tools.set("shell", Behavior::Ask("npm test".into())); + h.tools.set("ask", Behavior::AskQuestion); + let session = h.session().await; + let first = h + .runtime + .admit(&session, admit("first owner")) + .await + .unwrap(); + let initial = wait_pending(&h, &session, 1).await.remove(0); + assert_eq!( + initial.input_message_ids.as_slice(), + std::slice::from_ref(&first.message_id) + ); + let second = h + .runtime + .admit(&session, admit("second owner")) + .await + .unwrap(); + let third = h + .runtime + .admit(&session, admit("third owner")) + .await + .unwrap(); + assert_eq!( + h.runtime.pending_requests(Some(&session))[0] + .input_message_ids + .as_slice(), + std::slice::from_ref(&first.message_id) + ); + h.runtime + .reply_permission(&initial.id, PermissionReply::Once) + .await + .unwrap(); + let permission = wait_pending(&h, &session, 1).await.remove(0); + let cohort = vec![second.message_id, third.message_id]; + assert_eq!(permission.call_id, "c1"); + assert_eq!(permission.input_message_ids, cohort); + h.runtime.compact(&session, None).await.unwrap(); + h.runtime + .reply_permission(&permission.id, PermissionReply::Once) + .await + .unwrap(); + let question = wait_pending(&h, &session, 1).await.remove(0); + assert_eq!(question.call_id, "c2"); + assert_eq!(question.input_message_ids, cohort); + assert!(h.state(&session).await.compacted.is_some()); + h.runtime + .answer_question( + &question.id, + QuestionReply::Answers { + answers: vec![vec!["sqlite".into()]], + }, + ) + .await + .unwrap(); + h.settle(&session).await; + let event = h + .store + .read_events(&session, -1, 500) + .unwrap() + .events + .into_iter() + .find(|event| event.kind == "permission.asked.1") + .unwrap(); + assert_eq!( + event.data["input_message_ids"], + serde_json::json!([first.message_id]) + ); +} + +#[tokio::test] +async fn checked_out_input_keeps_request_ownership_when_newer_promotions_are_inactive() { + use cyber_server::runtime::BranchCheckout; + let h = Harness::new(Setup { + scripts: vec![( + "test/main", + vec![ + tools(&[("a", "shell", "{}")]), + text("a done"), + tools(&[("b", "shell", "{}")]), + text("b done"), + tools(&[("resumed", "shell", "{}")]), + text("resumed done"), + ], + )], + ..Setup::default() + }); + h.tools.set("shell", Behavior::Ask("npm test".into())); + let session = h.session().await; + let first = h + .runtime + .admit(&session, admit("first branch")) + .await + .unwrap(); + answer_permission(&h, &session).await; + h.runtime + .admit(&session, admit("newer input")) + .await + .unwrap(); + answer_permission(&h, &session).await; + h.runtime + .checkout( + &session, + &BranchCheckout { + message_id: Some(first.message_id.clone()), + exact: true, + ..BranchCheckout::default() + }, + ) + .await + .unwrap(); + h.runtime.resume(&session).await.unwrap(); + let request = wait_pending(&h, &session, 1).await.remove(0); + assert_eq!(request.call_id, "resumed"); + assert_eq!(request.input_message_ids, vec![first.message_id]); + h.runtime + .reply_permission(&request.id, PermissionReply::Once) + .await + .unwrap(); + h.settle(&session).await; +} +async fn answer_permission(h: &Harness, session: &str) { + let request = wait_pending(h, session, 1).await.remove(0); + h.runtime + .reply_permission(&request.id, PermissionReply::Once) + .await + .unwrap(); + h.settle(session).await; +} + +#[tokio::test] +async fn replay_rejects_a_step_input_cohort_from_another_active_path() { + let h = Harness::new(Setup::default()); + let session = h.session().await; + let mut events = h.store.read_events(&session, -1, 500).unwrap().events; + events.push(cyber_store::StoredEvent{id:"event_forged".into(),aggregate_id:session,seq:events.len() as i64,kind:"session.step.started.1".into(),data:serde_json::json!({"message_id":"msg_forged","provider":"test","model":"main","tools":false,"input_message_ids":["msg_other"]}),time_ms:0,causation_id:None}); + assert!(cyber_server::runtime::SessionState::replay(&events).is_err()); +} + #[tokio::test] async fn reject_with_feedback_continues_and_without_feedback_halts() { let h = Harness::new(Setup { diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 5a2ea4bb2..46299c3c1 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -85,11 +85,18 @@ cross-platform, SDK/plugin and full-workspace checks have not been rerun here. The local fix keeps these diagnostics nonfatal and adds a real-binary regression with unavailable Git snapshots. All seven JSONL tests and all three structured-output CLI tests pass after the fix. -2. **Major: request and interruption ownership is Session-wide.** - `Driver::asked` accepts every matching Session request and `interrupt` targets - the Session without binding to the driver's admitted messages. Track exact - input/request authority and test two drivers/owner clients sharing a Session, - including unrelated work present before driver startup. +2. **Partly fixed locally: direct request ownership; interruption remains Session-wide.** + Model-step events now capture the complete active input cohort before inference. + Direct permission/question requests expose that durable cohort for their exact + step/call. The driver collects, replies to and closes only requests whose whole + cohort belongs to its pending inputs. Actual shared-App controller regressions + cover two drivers, equal external correlation labels, foreign replies and close + while the other owner remains attached. Branch checkout and compaction retain + the captured proof; legacy/oversized evidence grants no inferred authority. + Session-wide interrupt and interactive presence, descendant/hook ownership, + background-job ownership and general event/result correlation remain required. + These direct-request checks do not accept task 6.2 or complete shared-Session + isolation. 3. **Partly fixed locally: completed history and blocking output.** Inputs now occupy at most 32 pending message-ID slots; emitted text/structured results retire into bounded counters and first-failure accounting. Admission diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index c8ca85b09..a908b063b 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -653,8 +653,9 @@ cyber exec --output-schema findings.schema.json --format json "audit the depende ``` - **Output:** the `init` line has `schema_version: 2`. Each prompt gets an `input` line when it is admitted (`status: admitted` or `refused`) and one `result` line when its answer ends, with its `id`, `message_id`, `result`, `error` and `exit_code`. Requests appear as `permission_request` and `question` lines. The run ends with one `end` line: the aggregate `exit_code`, `inputs`, usage, cost and duration. `--include-partial-messages` adds `text_delta` and `reasoning_delta` lines, and `--output-last-message ` writes the last answer's text to the file atomically when the run ends. -- **Requests:** the driver holds an interactive lease on the Session while it reads input, so the Session asks it instead of denying, and the run starts in `default` mode unless `--mode` says otherwise. Only requests this Session raised can be answered; any other `request_id` is an `error` line with `code: protocol`, and the run goes on. Nothing is ever approved for you. -- **Ending:** after `end` or EOF nothing new is admitted, requests still waiting are declined (`reject` for permissions, dismissed questions), and the process exits once every admitted prompt has its result and the Session is idle. +- **Requests:** the driver holds an interactive lease while reading input, and starts in `default` mode unless `--mode` says otherwise. Direct permission/question lines carry server-derived `input_message_ids`: the complete input cohort recorded for their exact assistant step and tool call. The driver accepts a request only for its own Session when every identity is one of its pending admitted inputs. Missing or legacy provenance stays unknown; a request ID from another input is a protocol error on reply. Nothing is ever approved automatically. +- **Ending:** after `end` or EOF nothing new is admitted, tracked owned requests are declined (`reject` for permissions, dismissed questions), and the process exits once every admitted prompt has its result and the Session is idle. +- **Shared-Session work still in progress:** scoped interruption, input-scoped interactive presence, descendant/hook request ownership and general result/event correlation remain required. The direct-request guard does not complete those guarantees. - **Limits and errors:** a line is at most 1 MiB and at most 32 prompts may wait for an answer. A line that is not a valid envelope, an over-long line or a 33rd waiting prompt is a fatal protocol error: an `error` line, no more input, the Session interrupted, and exit 2. - **Exit code:** 130 after SIGINT, 2 after a fatal protocol error, otherwise the first failed prompt's code in admission order (1), else 0. Failure to publish `--output-last-message` makes an otherwise successful run exit 1 and reports the error in `end`; per-input results remain available. Publication uses an exclusively created sibling file, replaces a destination symlink without writing its target, and removes its temporary file on failure. `--output-schema` applies to every prompt, whose `result` is then the validated object. - **Not combined with:** a prompt argument, `--command`, `--goal`, `--timeout`, `--max-cost` or `--max-tokens` (exit 2 before any Session). `--include-hook-events` is not available yet. diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 7bb95f7d4..88974948c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -36,6 +36,21 @@ Implement `--input-format stream-json` as a bounded asynchronous reader and prot Support the existing canonical user/text-or-parts/delivery, interrupt, permission_reply, question_reply and end frames. Require exact live request ownership for replies; the streaming transport does not fabricate browser/TUI presence and does not approve anything automatically. The stream's negotiated request/reply behavior may keep an owned ask pending only while that driver is live. On EOF/end stop accepting frames, drain acknowledged pending work in defined order, and terminate after all owned inputs settle; disconnect cannot leave asks waiting forever. Never cancel unrelated work in a reused Session. SIGINT follows owned interruption and exit 130. Lost acknowledgement and restart use request lookup; uncertain native outcomes retain their ordinary fences. +Request ownership must come from durable server evidence. Add optional +`input_message_ids` to pending permission/question requests. Capture the complete +active input cohort in each assistant-step event before inference, validate it at +runtime append and replay, then bind requests to that exact step and tool call. +Never infer ownership from globally latest promotion history: branch checkout can +make those promotions inactive. Active-path snapshots retain the cohort through +tool continuations and compaction; later steering cannot reassign an older call. +Bound proof to 32 identities +of at most 256 bytes each; missing, malformed or oversized evidence grants no +driver ownership. Historical steps/requests without this field remain readable with +unknown provenance. Direct request reply/close guards require every cohort identity +to be a pending input of this driver and the exact Session. Scoped interruption, +input-scoped presence, descendant ownership and other event/result correlation +remain separate implementation obligations. + Emit schema_version 2, one result per admitted input with stable correlation, optional partial text/reasoning and hook events, and atomic `--output-last-message`. Session-level usage remains separate from per-input results and descendant usage. Redact errors and bound output buffering; define aggregate exits as 130 for explicit SIGINT, 2 for a fatal protocol error, otherwise the first nonzero input outcome in admission order (0 only when all outcomes succeed). A failing input does not discard already acknowledged later input; fatal protocol errors stop new admission and settle or explicitly cancel only owned work. Cover these precedence rules in protocol tests. Do not silently change text-mode output or the existing review/workflow/Goal command contracts; unsupported option combinations fail before creating work. ## Delivery and acceptance diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md index a8a631f59..fae176aec 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/exec-mode/spec.md @@ -3,6 +3,20 @@ ### Requirement: Bounded interactive JSONL driver (P1) `--input-format stream-json` SHALL parse bounded typed user/control/reply/end envelopes independently of model execution, retain durable admission identity and emit one correlated terminal result per admitted input under schema version 2. Replies SHALL target only live requests owned by that driver. EOF/end SHALL stop new admission and drain acknowledged owned work without leaving unattended asks pending indefinitely or cancelling unrelated Session work. Malformed, oversized and over-capacity frames SHALL produce explicit protocol errors rather than silent text admission. `--output-schema` SHALL use root structured results; unstructured stdin behavior SHALL remain compatible. +#### Scenario: Direct model requests retain their complete input cohort +- **WHEN** a model step starts on the active branch +- **THEN** its server-recorded cohort SHALL contain every input driving that step, preserve it across tool continuations and compaction, and exclude inactive branch promotions +- **AND** runtime append and replay SHALL refuse contradictory nonempty provenance; legacy or oversized cohorts SHALL grant no inferred ownership +- **WHEN** the step's exact tool call opens a permission or question request +- **THEN** the pending request SHALL expose its captured `input_message_ids`, bounded to 32 identities of at most 256 UTF-8 bytes each + +#### Scenario: Two drivers have pending input in one Session +- **GIVEN** the first driver owns a live direct request and the second driver has admitted a different input +- **WHEN** the second driver sees that request, tries to reply, or closes while the first driver remains attached +- **THEN** it SHALL neither retain the request as its own nor reply to or dismiss it +- **AND** the first driver SHALL retain normal reply authority, while a closed driver SHALL decline only its own later direct requests +- **AND** user-provided correlation labels SHALL NOT grant request ownership + #### Scenario: Interrupt while inference runs - **WHEN** an active JSONL driver sends an interrupt while its model request is streaming - **THEN** the reader processes it without waiting for model completion and interruption applies only to the driver's owned execution diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 2b160f1db..e4fff571d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -142,7 +142,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. ## 6. Streaming JSONL input - [x] 6.1 Define typed version-2 envelopes, correlation, resource bounds, aggregate exit rules and driver-owned interactive request lifetime; document compatibility and unsupported combinations. -- [ ] 6.2 Implement concurrent bounded parsing/admission and exact owned permission/question replies, retained request lookup, interrupt and end/EOF draining without stopping unrelated work. Basic parsing/admission merged in #117. The 2026-11-09 code review found that request collection and interrupt target the Session without checking the driver's admitted message ownership, and completed inputs retain their full result payloads indefinitely. Complete those ownership/resource bounds and their shared-Session/long-run regressions before accepting this task. +- [ ] 6.2 Implement concurrent bounded parsing/admission and exact owned permission/question replies, retained request lookup, interrupt and end/EOF draining without stopping unrelated work. Basic parsing/admission merged in #117. Local continuations retire completed payloads and bind direct model requests to the complete recorded input cohort. Session-wide interruption/presence, descendant and background ownership, general event/result correlation and remaining resource bounds still need implementation and acceptance; the direct-request guard does not complete this task. - [ ] 6.3 Emit per-input results, optional partial/hook events and atomic last-message output with bounded backpressure and secret-safe errors. `--include-hook-events` is not implemented: hook receipts are deliberately not on the Session stream, so the lines would have nothing reliable to carry. - [ ] 6.4 Test multiple inputs, steering/queueing, malformed/oversized input, capacity, stale/cross-request replies, EOF while awaiting input, slow stdout, SIGINT, server restart and lost admission acknowledgement. Done: multiple inputs in order, per-input structured results, structured/plain steering while inference is gated, long-run slot retirement, malformed input (exit 2), unknown request replies, EOF while a request waits, unsupported combinations, with the real binary (`cyber-cli/tests/exec_stream_input.rs`). Open: shared-Session ownership, capacity, slow stdout, SIGINT, server restart and lost admission acknowledgement. @@ -1475,3 +1475,52 @@ permanent root removal still needs a separately validated detached authority and coordination path; the current checks were not relaxed or replaced, and this remaining capability has not been marked complete. M6/M8, parent tasks and issue #93 remain open. + + +Direct JSONL request ownership continuation (2026-11-10): assistant-step events +now capture the complete active-path input cohort before inference. Runtime append +and replay validate nonempty proof against branch-aware state. Private step +snapshots preserve it across continuations and compaction without extending the +public snapshot schema. Pending requests expose optional `input_message_ids` +from the exact recorded step/tool call, never from the latest global promotion. +Legacy or oversized evidence remains unknown; proof is bounded to 32 identities +of at most 256 bytes, with no partial cohort accepted. The driver requires every +identity to belong to its pending inputs before collecting/replying/closing a +direct request. External correlation labels confer no authority. OpenAPI and +TypeScript types are regenerated. + +Actual shared-App/Session controller tests cover permission and question requests, +foreign reply refusal, close while the first owner remains attached, normal owner +reply and a closed driver's own later request. Removing the guard reproduces the +foreign-request collection failure; the failed fixture is retained. Separate real +binary tests verify request proof and stdin permission/question replies. Server +regressions cover co-promoted inputs, later steering, actual compaction, inactive +branch promotions and contradictory replay. Runtime append preflights step proof; +raw Store append is not claimed to validate SessionState. + +Verification passes: 3 provenance unit tests; 443 server library tests with one +explicit subprocess-fixture test ignored; 22 request tests; branch checkout (1), +context compaction (19) and structured-root (16) integration tests; 33 exec unit +tests and 12 JSONL binary tests. Server/CLI all-target Clippy passes with warnings +denied. OpenAPI exact check and SDK generated-file check pass; SDK typecheck +and tests pass (136 passed, 10 configured external skips). These are local checks, +not full-workspace or current-source cross-platform acceptance. The native workflow +now independently runs provenance/request regressions alongside JSONL tests and +retains failed shared-App fixtures under the short artifact root. + +Focused code-review-skill review finds no outstanding finding in this direct +request increment. It follows the existing runtime/Store/CLI boundaries; SQL uses +bound parameters, unknown proof grants no driver ownership, replay preserves +legacy readability, and generated API changes are additive. The new producer, +fold, lookup and driver guard have unit/integration coverage. Whole-branch review +and current-source native acceptance remain required. Session-wide interruption +and interactive leases, descendant/hook requests, input-scoped background jobs, +output backpressure and broader event/result correlation remain implementation +gaps. No claim is made that releasing a Session-wide lease cannot affect another +owner without its own attached presence. Tasks 6.2–6.4 and issue #93 remain open. + +Final checkpoint checks: formatting, diff whitespace, workflow YAML and the +independent JSONL step/retained-root assertions pass. The CI-pinned OpenSpec +1.13.2 validates 100/100 items; cross-spec lint reports 0 errors and 25 warnings. +Other cached OpenSpec versions were not used for acceptance. The SDK generation +check is clean. No parent task is checked off by this increment. diff --git a/sdk/openapi.json b/sdk/openapi.json index cfbfa0056..47967ac34 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -9912,6 +9912,13 @@ "message_id": { "type": "string" }, + "input_message_ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Complete input cohort driving this exact recorded step/call. Empty means\nunknown; clients must not infer ownership from current Session state." + }, "origin": { "anyOf": [ { diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 1a80fb787..cdc6d91ab 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -3644,6 +3644,11 @@ export type PendingRequest = { session_id: string; call_id: string; message_id: string; + /** + * Complete input cohort driving this exact recorded step/call. Empty means + * unknown; clients must not infer ownership from current Session state. + */ + input_message_ids?: string[]; origin?: RequestOrigin | null; routed_to?: RequestRoute[]; } & ((PermissionAsk & { From 6bd2f57a6f04db86166645270d783f297c46a4b1 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 07:08:00 -0400 Subject: [PATCH 73/87] Keep managed cut fixtures within native socket path limits --- .../cyber-app/tests/support/managed_retirement_cut.rs | 4 +++- docs/goal-delivery.md | 7 +++++++ .../changes/complete-daily-coding-surfaces/tasks.md | 11 +++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/cyber-app/tests/support/managed_retirement_cut.rs b/crates/cyber-app/tests/support/managed_retirement_cut.rs index d33f40a58..2afa46ae8 100644 --- a/crates/cyber-app/tests/support/managed_retirement_cut.rs +++ b/crates/cyber-app/tests/support/managed_retirement_cut.rs @@ -188,7 +188,9 @@ async fn kill_at_cut(control: &Path, cut: &str) { ]) .env("CYBER_MANAGED_METADATA_CUT_CONTROL", control) .env("CYBER_MANAGED_METADATA_CUT", cut) - .env("TMPDIR", control) + // Inherit the short CI fixture root. Nesting another TempDir under the + // control directory exceeds Linux's Unix socket limit during worktree + // setup. The child retains its own fixture and reports its actual root. .stdout(log.try_clone().unwrap()) .stderr(log) .spawn() diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 46299c3c1..5c9aac8d3 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -59,6 +59,13 @@ code changes. The workflow also runs the four zero-payload CLI cases and retains their log. This resolves the inspected fixture routing problem in configuration; fresh-source native execution is still required to prove acceptance. +Run 42790242409 at `af0d10ed` passes both macOS jobs but still fails managed +process-cut setup on Linux: overriding the child's `TMPDIR` with the control +directory adds another temporary-directory component. The child now inherits +the short workflow root. Its fixture independently retains failures and records +its actual root for reopening; both sibling roots remain covered by artifact +upload. Linux execution of this correction remains required. + ### Location continuity acceptance Location continuity acceptance now exercises six combinations of missing, diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index e4fff571d..b5916c117 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1524,3 +1524,14 @@ independent JSONL step/retained-root assertions pass. The CI-pinned OpenSpec 1.13.2 validates 100/100 items; cross-spec lint reports 0 errors and 25 warnings. Other cached OpenSpec versions were not used for acceptance. The SDK generation check is clean. No parent task is checked off by this increment. + +Native CI follow-up: run 42790242409 at af0d10ed passes macOS ARM/Intel +but fails both Linux jobs during the managed metadata child fixture setup, before +its observed cut. The log reports SUN_LEN at the nested control/fixture root. +The child now inherits the workflow's short TMPDIR instead of nesting beneath +the control directory. It still retains its own failure root and writes the actual +root into fixture.json for reopening; the workflow uploads both roots. The actual +managed metadata process-death selection passes locally after this correction +(two entries, including the inert child entry and parent exercising both cuts). +Formatting and diff checks pass. Linker unwind and native C deprecation warnings +were observed without assigning origin. Linux execution remains required. From 9d0f6dcb0bb7519685cd15e8facdb7a57b350eb1 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 07:53:43 -0400 Subject: [PATCH 74/87] Apply trusted configuration limits to instruction loading --- .github/workflows/non-git-snapshot-proof.yml | 25 ++++++ crates/cyber-app/src/host.rs | 7 ++ crates/cyber-app/tests/instructions.rs | 29 +++++++ crates/cyber-cli/src/commands/doctor.rs | 70 +++++++++++++++- crates/cyber-core/src/config/gate.rs | 1 + crates/cyber-core/src/config/instructions.rs | 42 ++++++++++ crates/cyber-core/src/config/mod.rs | 2 + crates/cyber-core/src/config/validate.rs | 2 + crates/cyber-core/tests/config.rs | 82 +++++++++++++++++++ crates/cyber-server/src/runtime/context.rs | 45 ++++++++-- crates/cyber-server/src/runtime/drain.rs | 8 +- crates/cyber-server/src/runtime/host.rs | 9 ++ .../cyber-server/src/runtime/instructions.rs | 27 ++++-- .../cyber-server/tests/context_compaction.rs | 24 ++++++ crates/cyber-server/tests/support/mod.rs | 9 ++ crates/cyber-tools/src/host.rs | 8 ++ .../cyber-tools/tests/instruction_config.rs | 24 ++++++ docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 6 +- .../complete-daily-coding-surfaces/design.md | 2 +- .../specs/configuration/spec.md | 11 ++- .../specs/system-context/spec.md | 25 ++++++ .../complete-daily-coding-surfaces/tasks.md | 41 ++++++++++ 23 files changed, 474 insertions(+), 27 deletions(-) create mode 100644 crates/cyber-core/src/config/instructions.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 32291dd25..0ac1b6e70 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -25,6 +25,16 @@ on: - crates/cyber-server/src/http/sessions.rs - crates/cyber-app/tests/snapshot_git_runtime.rs - crates/cyber-app/tests/snapshot_reclamation.rs + - crates/cyber-app/tests/instructions.rs + - crates/cyber-core/src/config/** + - crates/cyber-core/tests/config.rs + - crates/cyber-tools/src/host.rs + - crates/cyber-tools/tests/instruction_config.rs + - crates/cyber-server/src/runtime/context.rs + - crates/cyber-server/src/runtime/instructions.rs + - crates/cyber-server/src/runtime/host.rs + - crates/cyber-server/tests/context_compaction.rs + - crates/cyber-server/tests/support/mod.rs - crates/cyber-cli/Cargo.toml - crates/cyber-cli/src/** - crates/cyber-cli/tests/branches_cli.rs @@ -174,6 +184,20 @@ jobs: run: | set -euo pipefail cargo test --locked -p cyber-server -p cyber-cli --features cyber-cli/native-git-acceptance --test structured_root --test exec_output_schema -- --nocapture 2>&1 | tee structured-result-proof.log + - name: Instruction limits and context observation regressions + if: ${{ !cancelled() }} + env: + TMPDIR: /tmp/c${{ github.run_number }} + shell: bash + run: | + set -euo pipefail + mkdir -p "$TMPDIR" + cargo test --locked -p cyber-core --test config instruction_limits -- --nocapture 2>&1 | tee instruction-limits-proof.log + cargo test --locked -p cyber-server --lib runtime::context:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-server --test context_compaction -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-app --features native-git-acceptance --test instructions -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-tools --test instruction_config -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::doctor::instruction_tests:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log - name: Retain native evidence if: always() uses: actions/upload-artifact@v4 @@ -193,6 +217,7 @@ jobs: zero-payload-retirement-proof.log jsonl-driver-proof.log structured-result-proof.log + instruction-limits-proof.log if-no-files-found: warn - name: Retain failed CLI fixture evidence if: ${{ failure() && !cancelled() }} diff --git a/crates/cyber-app/src/host.rs b/crates/cyber-app/src/host.rs index 3b8787b54..ab7b18ff5 100644 --- a/crates/cyber-app/src/host.rs +++ b/crates/cyber-app/src/host.rs @@ -76,6 +76,13 @@ impl ToolHost for AppHost { self.builtin.claude_compat(directory) } + fn instruction_limits( + &self, + directory: &str, + ) -> Result { + self.builtin.instruction_limits(directory) + } + fn language(&self, directory: &str) -> Option { self.builtin.language(directory) } diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index 93a6e88a4..e5663ea67 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -41,6 +41,35 @@ fn reply(body: &Value) -> Value { json!({"content": "ok"}) } +#[tokio::test] +async fn configured_instruction_limits_bound_baseline_and_nested_provider_text() { + let f = Fixture::start_with( + json!({"instruction_limits":{"max_file_bytes":16,"max_total_bytes":24}}), + ) + .await; + f.write("AGENTS.md", &"λ".repeat(20)); + f.write("CYBER.md", &"Ж".repeat(20)); + f.write("web/AGENTS.md", &"Ψ".repeat(20)); + f.write("web/CYBER.md", &"Ω".repeat(20)); + f.write("web/src/app.ts", "export const app = true;"); + let session = f.session(&f.repo).await; + f.prompt(&session, &f.repo, "read the app first", 2).await; + let body = f.last_request(); + let baseline = system_prompt(&body); + assert!(baseline.contains(&"λ".repeat(8)), "{baseline}"); + assert!(!baseline.contains(&"λ".repeat(9)), "{baseline}"); + assert!(baseline.contains(&"Ж".repeat(4)), "{baseline}"); + assert!(!baseline.contains(&"Ж".repeat(5)), "{baseline}"); + assert!(baseline.contains("instruction_limits.max_file_bytes (16 bytes)")); + let nested = last_tool_result(&body); + assert!(nested.contains(&"Ψ".repeat(8)), "{nested}"); + assert!(!nested.contains(&"Ψ".repeat(9)), "{nested}"); + assert!(nested.contains(&"Ω".repeat(4)), "{nested}"); + assert!(!nested.contains(&"Ω".repeat(5)), "{nested}"); + assert!(nested.contains("total budget")); + f.app.runtime.shutdown().await; +} + struct Fixture { _temp: tempfile::TempDir, repo: PathBuf, diff --git a/crates/cyber-cli/src/commands/doctor.rs b/crates/cyber-cli/src/commands/doctor.rs index ac834ce42..a3bd558e0 100644 --- a/crates/cyber-cli/src/commands/doctor.rs +++ b/crates/cyber-cli/src/commands/doctor.rs @@ -128,14 +128,27 @@ fn commands_check(catalog: &Value) -> Check { /// The instruction files the next Session at this Location loads, in load order, with a /// warning for truncated files and unresolved or refused imports. `config` is `None` when -/// the Location's config failed to load: like the runtime, `CLAUDE.md` is then off and no -/// project file passes the read gate. +/// the Location's config failed to load: like the runtime, no instruction file is read +/// using fallback limits. fn instructions(ctx: &Context, config: Option<&cyber_core::config::Resolved>) -> Check { use cyber_server::runtime::instructions::{InstructionSettings, load_baseline}; // The same per-Location `compat.claude` and `read` rules the runtime reads through its // tool host. let compat = config.is_some_and(|c| cyber_core::config::claude_compat(&c.value)); let mut settings = InstructionSettings::from_env(compat, &ctx.env); + let limits = config + .ok_or_else(|| "configuration unavailable".to_owned()) + .and_then(|c| cyber_core::config::InstructionLimits::from_config(&c.value)); + match limits { + Ok(limits) => settings.set_limits(limits), + Err(reason) => { + return check( + "instructions", + Status::Warn, + format!("unavailable: {reason}"), + ); + } + } settings.read_gate = Some(cyber_tools::permissions::instruction_read_gate( config.map(|c| (&c.value, &c.sources)), &ctx.location, @@ -495,3 +508,56 @@ async fn account_check(ctx: &Context) -> Check { Err(error) => check("account", Status::Warn, error.to_string()), } } + +#[cfg(test)] +mod instruction_tests { + use super::*; + use clap::Parser; + + #[test] + fn doctor_uses_configured_limits_and_does_not_load_without_configuration() { + let temp = tempfile::tempdir().unwrap(); + let env = std::collections::HashMap::from([( + "CYBER_HOME".to_owned(), + temp.path().join("cyber").display().to_string(), + )]); + let cli = crate::cli::Cli::try_parse_from(["cyber", "doctor"]).unwrap(); + let mut ctx = Context::existing(&cli.global).unwrap(); + ctx.home = temp.path().join("home"); + ctx.paths = cyber_core::paths::Paths::resolve(&env, &ctx.home); + ctx.paths.ensure().unwrap(); + ctx.location = temp.path().join("repo"); + std::fs::create_dir_all(ctx.location.join(".git")).unwrap(); + std::fs::write(ctx.paths.config.join("AGENTS.md"), "λ".repeat(40)).unwrap(); + std::fs::write( + ctx.paths.config.join("cyber.json"), + r#"{"instruction_limits":{"max_file_bytes":16,"max_total_bytes":24}}"#, + ) + .unwrap(); + let resolved = cyber_core::config::load(&cyber_core::config::LoadRequest { + location: &ctx.location, + paths: &ctx.paths, + env: &env, + home: &ctx.home, + profile: None, + overrides: &[], + flags: json!({}), + }) + .unwrap(); + let check = instructions(&ctx, Some(&resolved)); + assert_eq!(check.status, Status::Warn); + assert!( + check + .detail + .contains("instruction_limits.max_file_bytes (16 bytes)"), + "{}", + check.detail + ); + // An unreadable instruction must not be inspected when configuration fails. + std::fs::remove_file(ctx.paths.config.join("AGENTS.md")).unwrap(); + std::fs::create_dir(ctx.paths.config.join("AGENTS.md")).unwrap(); + let check = instructions(&ctx, None); + assert_eq!(check.status, Status::Warn); + assert_eq!(check.detail, "unavailable: configuration unavailable"); + } +} diff --git a/crates/cyber-core/src/config/gate.rs b/crates/cyber-core/src/config/gate.rs index 87f71e3d1..ec45a1c0c 100644 --- a/crates/cyber-core/src/config/gate.rs +++ b/crates/cyber-core/src/config/gate.rs @@ -10,6 +10,7 @@ use serde_json::{Map, Value}; /// Top-level keys that configure executable integrations, endpoints or boundaries. const SENSITIVE_KEYS: &[&str] = &[ "budgets", + "instruction_limits", "providers", "mcp", "browser", diff --git a/crates/cyber-core/src/config/instructions.rs b/crates/cyber-core/src/config/instructions.rs new file mode 100644 index 000000000..fba572c37 --- /dev/null +++ b/crates/cyber-core/src/config/instructions.rs @@ -0,0 +1,42 @@ +//! Byte budgets for privileged instruction assembly. `instructions` stays an array. +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const DEFAULT_MAX_FILE_BYTES: usize = 32 * 1024; +pub const DEFAULT_MAX_TOTAL_BYTES: usize = 128 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct InstructionLimits { + pub max_file_bytes: usize, + pub max_total_bytes: usize, +} + +impl Default for InstructionLimits { + fn default() -> Self { + Self { + max_file_bytes: DEFAULT_MAX_FILE_BYTES, + max_total_bytes: DEFAULT_MAX_TOTAL_BYTES, + } + } +} + +impl InstructionLimits { + pub fn from_config(config: &Value) -> Result { + let Some(value) = config.get("instruction_limits") else { + return Ok(Self::default()); + }; + if !value.is_object() { + return Err("instruction_limits: expected an object".into()); + } + let limits: Self = serde_json::from_value(value.clone()) + .map_err(|error| format!("instruction_limits: {error}"))?; + if limits.max_file_bytes == 0 || limits.max_total_bytes == 0 { + return Err("instruction_limits: byte limits must be positive integers".into()); + } + if limits.max_file_bytes > limits.max_total_bytes { + return Err("instruction_limits.max_file_bytes must not exceed max_total_bytes".into()); + } + Ok(limits) + } +} diff --git a/crates/cyber-core/src/config/mod.rs b/crates/cyber-core/src/config/mod.rs index 37ee31c2a..ff62e6803 100644 --- a/crates/cyber-core/src/config/mod.rs +++ b/crates/cyber-core/src/config/mod.rs @@ -13,6 +13,7 @@ mod edit; mod gate; mod git; mod hooks; +mod instructions; mod intelligence; mod jsonc; mod load; @@ -52,6 +53,7 @@ pub use edit::{set_document_member, set_global_key}; pub use gate::{SensitiveSplit, canonical_json, split_sensitive}; pub use git::GitSettings; pub use hooks::{HookCondition, HookGroup, HookHandler, HookKind, HookSettings}; +pub use instructions::{DEFAULT_MAX_FILE_BYTES, DEFAULT_MAX_TOTAL_BYTES, InstructionLimits}; pub use intelligence::{FormatterConfig, FormatterSettings, LspServerConfig, LspSettings}; pub use jsonc::{parse as parse_jsonc, to_json as jsonc_to_json}; pub use load::{ diff --git a/crates/cyber-core/src/config/validate.rs b/crates/cyber-core/src/config/validate.rs index 92bc40866..a800548ad 100644 --- a/crates/cyber-core/src/config/validate.rs +++ b/crates/cyber-core/src/config/validate.rs @@ -27,6 +27,7 @@ const KNOWN_KEYS: &[&str] = &[ "skills", "commands", "instructions", + "instruction_limits", "references", "memory", "browser", @@ -132,6 +133,7 @@ fn settings_issues(value: &Value) -> Vec { .err() .map(|error| error.to_string()); let sections = [ + super::InstructionLimits::from_config(value).err(), crate::worktrees::Settings::from_config(value).err(), super::AutoModeSettings::from_config(value).err(), crate::untrusted::UntrustedSettings::from_config(value).err(), diff --git a/crates/cyber-core/tests/config.rs b/crates/cyber-core/tests/config.rs index e2899d711..6154700e8 100644 --- a/crates/cyber-core/tests/config.rs +++ b/crates/cyber-core/tests/config.rs @@ -490,6 +490,88 @@ fn instructions_concatenate_global_first() { ); } +#[test] +fn instruction_limits_layer_by_member_without_changing_instruction_arrays() { + use config::InstructionLimits; + let f = Fixture::new(); + assert_eq!( + InstructionLimits::from_config(&f.load().unwrap().value).unwrap(), + InstructionLimits::default() + ); + f.write("global:cyber.json", r#"{"instructions":["global.md"],"instruction_limits":{"max_file_bytes":16,"max_total_bytes":64}}"#); + f.write("cyber.jsonc", r#"{"instructions":["project.md"],"instruction_limits":{"max_total_bytes":32},"profiles":{"small":{"instruction_limits":{"max_file_bytes":12}}}}"#); + let held = f.load().unwrap(); + assert!(!held.trust.trusted); + assert_eq!( + InstructionLimits::from_config(&held.value).unwrap(), + InstructionLimits { + max_file_bytes: 16, + max_total_bytes: 64 + } + ); + f.approve_current(); + let layered = f.load().unwrap(); + assert_eq!( + InstructionLimits::from_config(&layered.value).unwrap(), + InstructionLimits { + max_file_bytes: 16, + max_total_bytes: 32 + } + ); + assert_eq!( + layered.value["instructions"], + json!(["global.md", "project.md"]) + ); + assert!( + !layered + .warnings + .iter() + .any(|s| s.contains("unknown config key \"instruction_limits\"")) + ); + assert!(layered.sources["/instruction_limits/max_total_bytes"].starts_with("project:")); + let mut request = f.request(&f.repo, &[]); + request.profile = Some("small"); + assert_eq!( + InstructionLimits::from_config(&config::load(&request).unwrap().value) + .unwrap() + .max_file_bytes, + 12 + ); + let overrides = vec!["instruction_limits.max_file_bytes=8".into()]; + let resolved = config::load(&f.request(&f.repo, &overrides)).unwrap(); + assert_eq!( + InstructionLimits::from_config(&resolved.value).unwrap(), + InstructionLimits { + max_file_bytes: 8, + max_total_bytes: 32 + } + ); +} + +#[test] +fn instruction_limits_reject_invalid_values_before_configuration_is_usable() { + let f = Fixture::new(); + for value in [ + Value::Null, + json!(false), + json!([]), + json!(0), + json!({"max_file_bytes":0}), + json!({"max_total_bytes":-1}), + json!({"max_file_bytes":"16"}), + json!({"max_file_bytes":1.5}), + json!({"max_bytes":16}), + json!({"max_file_bytes":33,"max_total_bytes":32}), + ] { + f.write( + "global:cyber.json", + &json!({"instruction_limits":value}).to_string(), + ); + let error = f.load().unwrap_err().to_string(); + assert!(error.contains("instruction_limits"), "{value}: {error}"); + } +} + /// `provider-catalog` → Model behavior profiles: an invalid value warns and is ignored. #[test] fn an_invalid_model_profile_value_warns_naming_the_key() { diff --git a/crates/cyber-server/src/runtime/context.rs b/crates/cyber-server/src/runtime/context.rs index dc4172a53..1685bfe28 100644 --- a/crates/cyber-server/src/runtime/context.rs +++ b/crates/cyber-server/src/runtime/context.rs @@ -15,7 +15,8 @@ pub struct ContextInputs { pub directory: PathBuf, pub global_config_dir: PathBuf, pub shell: String, - pub instructions: super::instructions::InstructionSettings, + /// A configuration failure withholds instruction reads, independently of date/env. + pub instructions: Result, pub today: Option, } @@ -199,11 +200,17 @@ fn environment(inputs: &ContextInputs) -> String { /// `core/instructions` (see [`super::instructions::load_baseline`]), plus the files read, /// in load order. fn instructions(inputs: &ContextInputs) -> (Observed, Vec) { - let loaded = super::instructions::load_baseline( - &inputs.global_config_dir, - &inputs.directory, - &inputs.instructions, - ); + let loaded = inputs + .instructions + .as_ref() + .map_err(Clone::clone) + .and_then(|settings| { + super::instructions::load_baseline( + &inputs.global_config_dir, + &inputs.directory, + settings, + ) + }); match loaded { Err(reason) => (Observed::Unavailable(reason), Vec::new()), Ok(loaded) => { @@ -247,14 +254,14 @@ mod tests { directory: dir.to_path_buf(), global_config_dir: dir.join("no-global"), shell: "zsh".into(), - instructions: super::super::instructions::InstructionSettings { + instructions: Ok(super::super::instructions::InstructionSettings { claude_md: true, project: true, max_bytes: super::super::instructions::DEFAULT_MAX_BYTES, max_total_bytes: super::super::instructions::DEFAULT_MAX_TOTAL_BYTES, home: None, read_gate: None, - }, + }), today: Some("2026-10-03".into()), } } @@ -271,6 +278,28 @@ mod tests { assert!(text.contains("use pnpm") && !text.contains("use npm")); } + #[test] + fn unavailable_limits_withhold_files_without_hiding_date_or_environment() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("AGENTS.md")).unwrap(); + let mut inputs = inputs(dir.path()); + inputs.instructions = Err("instruction_limits unavailable".into()); + let observed = observe(&inputs); + assert_eq!( + observed.sources["core/instructions"], + Observed::Unavailable("instruction_limits unavailable".into()) + ); + assert_eq!( + observed.sources["core/date"], + Observed::Value("Today's date: 2026-10-03".into()) + ); + assert!(matches!( + observed.sources["core/environment"], + Observed::Value(_) + )); + assert!(observed.files["core/instructions"].is_empty()); + } + /// Regression: InstructionsLoaded `file_paths` came from parsing rendered text, /// so a checkout's AGENTS.md could forge `Instructions from:` lines. #[test] diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 999578179..c45585306 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -944,7 +944,8 @@ impl Inner { pub(crate) fn instruction_settings( &self, directory: &str, - ) -> super::instructions::InstructionSettings { + ) -> Result { + let limits = self.tools.instruction_limits(directory)?; let compat = self .tools .claude_compat(directory) @@ -954,7 +955,8 @@ impl Inner { &cyber_core::env::ProcessEnv, ); settings.read_gate = self.tools.instruction_read_gate(directory); - settings + settings.set_limits(limits); + Ok(settings) } pub(crate) fn context_inputs(&self, directory: &str) -> context::ContextInputs { @@ -2071,7 +2073,7 @@ impl Inner { return None; } let path = invocation.input.get("path")?.as_str()?; - let settings = self.instruction_settings(&invocation.directory); + let settings = self.instruction_settings(&invocation.directory).ok()?; let location = std::path::PathBuf::from(&invocation.directory); Some(ReadTarget { file: super::instructions::read_target(&location, settings.home.as_deref(), path), diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index 01cd977b6..b39facf5e 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -909,6 +909,15 @@ pub trait ToolHost: Send + Sync { None } + /// Validated byte budgets from the same resolved Location configuration. + /// A resolution error must not fall back to reading with different limits. + fn instruction_limits( + &self, + _directory: &str, + ) -> Result { + Ok(Default::default()) + } + /// The Location's `read` permission rules as a gate on project instruction files and /// their imports; `None` (a host without a permission policy) loads every one. fn instruction_read_gate(&self, _directory: &str) -> Option { diff --git a/crates/cyber-server/src/runtime/instructions.rs b/crates/cyber-server/src/runtime/instructions.rs index 56d48cabd..234bb6fda 100644 --- a/crates/cyber-server/src/runtime/instructions.rs +++ b/crates/cyber-server/src/runtime/instructions.rs @@ -25,10 +25,10 @@ pub const GLOBAL_FILES: [&str; 2] = ["AGENTS.md", "CYBER.md"]; /// Takes the `AGENTS.md` slot and replaces every file in [`OVERRIDDEN`] of its directory. pub const OVERRIDE_FILE: &str = "AGENTS.override.md"; const OVERRIDDEN: [&str; 3] = ["AGENTS.md", "CLAUDE.md", "CYBER.md"]; -/// `instructions.max_bytes`: each file is truncated here with a notice. -pub const DEFAULT_MAX_BYTES: usize = 32 * 1024; +/// `instruction_limits.max_file_bytes`: each file is truncated here with a notice. +pub const DEFAULT_MAX_BYTES: usize = cyber_core::config::DEFAULT_MAX_FILE_BYTES; /// All baseline instruction text together, imports included, stops here. -pub const DEFAULT_MAX_TOTAL_BYTES: usize = 4 * DEFAULT_MAX_BYTES; +pub const DEFAULT_MAX_TOTAL_BYTES: usize = cyber_core::config::DEFAULT_MAX_TOTAL_BYTES; /// The instruction text one nested read reminder carries; the rest is named, not attached. pub const NESTED_MAX_TOTAL_BYTES: usize = DEFAULT_MAX_BYTES; const MAX_IMPORT_DEPTH: usize = 5; @@ -86,6 +86,11 @@ impl PartialEq for ReadGate { impl Eq for ReadGate {} impl InstructionSettings { + pub fn set_limits(&mut self, limits: cyber_core::config::InstructionLimits) { + self.max_bytes = limits.max_file_bytes; + self.max_total_bytes = limits.max_total_bytes; + } + pub fn from_env(claude_compat: bool, env: &dyn EnvSource) -> Self { Self { claude_md: claude_compat @@ -263,7 +268,7 @@ pub fn load_nested( let mut loader = Loader::new(settings, settings.max_total_bytes); let _ = fill_baseline(&mut loader, global_dir, &location); loader.out = Instructions::default(); - loader.remaining = NESTED_MAX_TOTAL_BYTES; + loader.remaining = NESTED_MAX_TOTAL_BYTES.min(settings.max_total_bytes); loader.seen.extend(attached.iter().map(PathBuf::from)); let root = canonical(&cyber_core::config::project_root(&location)); for file in dirs.iter().flat_map(|dir| directory_files(dir, settings)) { @@ -413,11 +418,11 @@ impl<'a> Loader<'a> { let (notice, warning) = if limit == max { ( format!( - "[Truncated: {} exceeds instructions.max_bytes ({max} bytes); {omitted} bytes omitted]", + "[Truncated: {} exceeds instruction_limits.max_file_bytes ({max} bytes); {omitted} bytes omitted]", path.display() ), format!( - "{} exceeds instructions.max_bytes ({max} bytes) and was truncated", + "{} exceeds instruction_limits.max_file_bytes ({max} bytes) and was truncated", path.display() ), ) @@ -524,7 +529,7 @@ fn read_bounded(path: &Path, limit: usize) -> Result { return Err(irregular()); } let mut bytes = Vec::new(); - file.take(limit as u64 + 1) + file.take((limit as u64).saturating_add(1)) .read_to_end(&mut bytes) .map_err(|e| error(&e))?; let cut = bytes.len() > limit; @@ -811,7 +816,11 @@ mod tests { text.contains(&format!("\n{}\n[Truncated: ", "é".repeat(5))), "{text}" ); - assert!(text.contains("exceeds instructions.max_bytes (11 bytes); 70 bytes omitted]")); + assert!( + text.contains( + "exceeds instruction_limits.max_file_bytes (11 bytes); 70 bytes omitted]" + ) + ); assert_eq!(loaded.warnings.len(), 1); } @@ -981,6 +990,8 @@ mod tests { assert_eq!(read.omitted, 990); let read = read_bounded(&path, 1000).unwrap(); assert_eq!((read.text.len(), read.omitted), (1000, 0)); + let read = read_bounded(&path, usize::MAX).unwrap(); + assert_eq!((read.text.len(), read.omitted), (1000, 0)); } /// Regression: per-file truncation did not bound the whole, so a `CYBER.md` importing diff --git a/crates/cyber-server/tests/context_compaction.rs b/crates/cyber-server/tests/context_compaction.rs index f79855fcf..9a0443cf1 100644 --- a/crates/cyber-server/tests/context_compaction.rs +++ b/crates/cyber-server/tests/context_compaction.rs @@ -218,6 +218,30 @@ async fn unreadable_instructions_block_the_first_turn_and_keep_the_prompt() { assert_eq!(h.state(&id).await.inbox[0].status, InputStatus::Promoted); } +#[tokio::test] +async fn unavailable_instruction_limits_keep_the_first_input_retryable_without_inference() { + let h = Harness::new(Setup { + scripts: vec![("test/main", vec![text("ok")])], + ..Setup::default() + }); + std::fs::write(h.repo.join("AGENTS.md"), "configured instructions").unwrap(); + *h.tools.instruction_limits.lock().unwrap() = Err("instruction_limits unavailable".into()); + let mut live = h.runtime.subscribe(); + let id = h.session().await; + h.runtime.admit(&id, admit("hi")).await.unwrap(); + h.settle(&id).await; + let (kind, _) = next_error(&mut live).await; + assert_eq!(kind, "context_initialization_blocked"); + assert!(h.models.requests("test/main").is_empty()); + assert!(h.state(&id).await.epoch.is_none()); + assert_eq!(h.state(&id).await.inbox[0].status, InputStatus::Pending); + *h.tools.instruction_limits.lock().unwrap() = Ok(Default::default()); + h.runtime.wake(&id).await.unwrap(); + h.settle(&id).await; + assert_eq!(h.models.requests("test/main").len(), 1); + assert_eq!(h.state(&id).await.inbox[0].status, InputStatus::Promoted); +} + #[tokio::test] async fn provider_family_switch_starts_a_new_epoch() { let h = Harness::new(Setup { diff --git a/crates/cyber-server/tests/support/mod.rs b/crates/cyber-server/tests/support/mod.rs index 4498a9763..de66a3200 100644 --- a/crates/cyber-server/tests/support/mod.rs +++ b/crates/cyber-server/tests/support/mod.rs @@ -187,6 +187,7 @@ pub enum Behavior { } pub struct Tools { + pub instruction_limits: Mutex>, pub inference: Mutex, pub required_failure: Mutex>, pub defs: Mutex>, @@ -252,6 +253,7 @@ impl Tools { ("write".to_string(), Behavior::Return("written".into())), ]); Arc::new(Self { + instruction_limits: Mutex::new(Ok(Default::default())), inference: Mutex::new(AgentInference::default()), required_failure: Mutex::new(None), defs: Mutex::new(defs), @@ -299,6 +301,13 @@ impl LocationGuard for TestLocationGuard { } impl ToolHost for Tools { + fn instruction_limits( + &self, + _directory: &str, + ) -> Result { + self.instruction_limits.lock().unwrap().clone() + } + fn redact_model_text( &self, _directory: &str, diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index 36e9376db..dcdcb9fa9 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1137,6 +1137,14 @@ impl ToolHost for BuiltinHost { )) } + fn instruction_limits( + &self, + directory: &str, + ) -> Result { + let (config, _) = (self.opts.config)(Path::new(directory))?; + cyber_core::config::InstructionLimits::from_config(&config) + } + fn finalize_skill_output( &self, directory: &str, diff --git a/crates/cyber-tools/tests/instruction_config.rs b/crates/cyber-tools/tests/instruction_config.rs index 500974d4f..ff6e3701a 100644 --- a/crates/cyber-tools/tests/instruction_config.rs +++ b/crates/cyber-tools/tests/instruction_config.rs @@ -44,6 +44,7 @@ fn a_config_that_fails_to_load_turns_claude_md_off_and_refuses_project_files() { let file = root.join("CYBER.md"); let broken = host(&root, Err("ConfigParseError: cyber.json:1:3".into())); assert_eq!(broken.claude_compat(&directory), Some(false)); + assert!(broken.instruction_limits(&directory).is_err()); let gate = broken.instruction_read_gate(&directory).unwrap(); assert!(!gate.allows(&file, &file)); let valid = host(&root, Ok(json!({}))); @@ -53,3 +54,26 @@ fn a_config_that_fails_to_load_turns_claude_md_off_and_refuses_project_files() { let env = root.join(".env"); assert!(!gate.allows(&env, &env)); } + +#[test] +fn the_host_reports_configured_limits_and_refuses_invalid_limits() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let directory = root.to_str().unwrap(); + let valid = host( + &root, + Ok(json!({"instruction_limits":{"max_file_bytes":16,"max_total_bytes":24}})), + ); + assert_eq!( + valid.instruction_limits(directory).unwrap(), + cyber_core::config::InstructionLimits { + max_file_bytes: 16, + max_total_bytes: 24 + } + ); + let invalid = host( + &root, + Ok(json!({"instruction_limits":{"max_file_bytes":0}})), + ); + assert!(invalid.instruction_limits(directory).is_err()); +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 5c9aac8d3..5a85c3431 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Implement trusted path-rule discovery/precedence/bounds and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now use configured file/total text budgets, including lower nested limits. Complete rendered assembly accounting, trusted path-rule discovery/precedence and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index a908b063b..937222bab 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1190,16 +1190,16 @@ Keep it short and specific to Cyber Code, and leave shared rules in `AGENTS.md`: - **Imports.** A line containing only `@path` inlines that file after its importer, as its own block. Paths resolve relative to the importing file, or to home for `@~/`. Lines inside code fences are ignored, imports nest up to a depth of 5, and a cycle is skipped with a warning naming both files. - **Project files stay in the project.** A project instruction file, or an import from one, that resolves (after symlinks) outside the project root is not loaded and produces a warning, so a checkout cannot pull `~/.ssh/...` or `../other-repo` into the context, whether through `@path` or through a symlinked `CYBER.md`. A symlink inside the project (`CYBER.md -> AGENTS.md`) still works. Global files are yours and may import anywhere. Unresolved imports are warnings, not errors. -- **Project files follow your `read` rules.** A project instruction file or import loads only when your `read` permission rules allow it outright. One they ask about or deny is skipped with a warning and no prompt, so under the default rules `@.env` (or a `CYBER.md` symlinked to `.env`) never reaches the context, and neither does a file a `read` deny covers. If the project's config does not parse, no project instruction file loads and `CLAUDE.md` is treated as off until it is fixed. +- **Project files follow your `read` rules.** A project instruction file or import loads only when your `read` permission rules allow it outright. One they ask about or deny is skipped with a warning and no prompt, so under the default rules `@.env` (or a `CYBER.md` symlinked to `.env`) never reaches the context, and neither does a file a `read` deny covers. If the Location's config cannot be resolved, instruction observation is unavailable until it is fixed; the first Turn remains retryable without inference, and an existing Epoch retains its last available instruction snapshot. - **Only regular files.** A FIFO, device or directory in place of an instruction file is never opened for reading. In the baseline it makes the instructions unavailable, so the first Turn is blocked as for any unreadable instruction file (`cyber doctor` names it); found by a nested read, it is skipped with a warning. -- **Size.** Each file is cut at 32768 bytes (`instructions.max_bytes`, not configurable yet) with a `[Truncated: ...]` notice, and reading stops there, however large the file is. The whole baseline, imports included, stops at 131072 bytes: the file that crosses it is cut with a notice, and later files are left out with a warning. `cyber doctor` shows both. +- **Size.** `instruction_limits` accepts `max_file_bytes` (default 32768) and `max_total_bytes` (default 131072). Both must be positive integers, with the file limit no larger than the total. Each file's text is cut at the file limit with a `[Truncated: ...]` notice. Baseline text, imports included, consumes the total budget; later files are named as omitted. Labels and diagnostic notices are additional text. `cyber doctor` uses the same settings and reports truncation. Project definitions are withheld by the existing workspace-trust gate until approved, including definitions inside profiles. The object follows normal configuration layering by member, including profiles and command-line overrides; it does not change the existing concatenated `instructions` array or introduce a new managed-policy key. - **Edits mid-session** reach the model at the next Safe Boundary as one durable context update; the cached prompt prefix is not rewritten. ### Nested files on read Files in directories below the Location are not in the baseline. When the built-in `read` tool successfully reads a file under such a directory, that directory's instruction files (and those of the directories between) are appended to the result inside a `` block, once per file per Context Epoch; the attachment is recorded durably with the tool settlement, and a new epoch (for example after compaction) starts over. Reads through `bash`, `grep` or MCP tools do not trigger it, and neither do denied or failed reads. -One reminder carries at most 32768 bytes of instruction text. Files past that are named in the reminder (`read them with the read tool`) instead of attached. When even that reminder does not fit the tool output budget (`tool_output`), the read gets a short list of the file paths; when nothing fits, the read keeps its own output and the files can attach on a later read. Instruction files never turn a successful read into an error. Each attachment fires the `InstructionsLoaded` hook with `load_reason: "nested_read"` and the attached `file_paths`, as the baseline does with `session_start` and `changed`. +One reminder carries at most the smaller of 32768 bytes and `instruction_limits.max_total_bytes` of instruction text. Files past that are named in the reminder (`read them with the read tool`) instead of attached. When even that reminder does not fit the tool output budget (`tool_output`), the read gets a short list of the file paths; when nothing fits, the read keeps its own output and the files can attach on a later read. Instruction files never turn a successful read into an error. Each attachment fires the `InstructionsLoaded` hook with `load_reason: "nested_read"` and the attached `file_paths`, as the baseline does with `session_start` and `changed`. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 88974948c..db560c3af 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -22,7 +22,7 @@ Use the canonical 200 MiB per-Location fallback limit and seven-day default rete Load `.cyber/rules/*.md` and enabled `.claude/rules/*.md` at the existing directory precedence. Rules without `paths` join the baseline. Relative `paths` globs activate after a successful authorized read/edit/patch of a matching path; denied/failed operations do not consume once-per-Epoch delivery. Successful partial file settlements retain correct reminder ownership. Bind project root, definition digest, trust/config generation, successful operation identity and Context Epoch. Keep existing directory instruction ordering and `AGENTS.override.md` replacement semantics; do not use a project rule to widen tool/permission authority. -Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. +Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. Rendered labels, notices and omitted-file inventories still need full assembly accounting under task 4.1. ### 4. Root structured output is per admitted prompt diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/configuration/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/configuration/spec.md index 2ebfc3e5d..6cad7ca6a 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/configuration/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/configuration/spec.md @@ -1,7 +1,7 @@ ## MODIFIED Requirements ### Requirement: Top-level keys -(P0) The schema SHALL accept the top-level keys `$schema`, `model`, `small_model`, `model_roles`, `fallback_models`, `providers`, `agents`, `default_agent`, `mode`, `output_style` (`system-context` → Output style selection), `permissions`, `sandbox`, `hooks`, `plugins`, `mcp`, `mcp_serve` (MCP serve; global configuration only), `skills`, `commands`, `instructions`, `instruction_limits`, `references`, `memory`, `browser`, `computer`, `compaction`, `snapshots`, `worktrees`, `lsp`, `formatters`, `tool_output`, `workflows`, `goals`, `teams`, `loops`, `messaging`, `peers`, `remote`, `runners`, `channels`, `account`, `share`, `telemetry`, `budgets`, `policy`, `profiles`, `default_profile`, `shell`, `attachments`, `background`, `tools`, `git`, `ide`, `autofix`, `network`, `storage`, `compat`, `services`, `features`, `experimental` (deprecated alias of `features`) and `autoupdate` (`installation-upgrade`). Collection keys SHALL use plural names, and entries SHALL use `disabled: true` to turn off an inherited entry. +(P0) The schema SHALL accept the top-level keys `$schema`, `model`, `small_model`, `model_roles`, `fallback_models`, `fallback` (`provider-catalog` → Model fallback chain), `providers`, `agents`, `default_agent`, `mode`, `output_style` (`system-context` → Output style selection), `language` (`localization`), `permissions`, `sandbox`, `hooks`, `plugins`, `mcp`, `mcp_serve` (MCP serve; global configuration only), `skills`, `commands`, `instructions`, `instruction_limits`, `references`, `memory`, `browser`, `computer`, `compaction`, `snapshots`, `worktrees`, `lsp`, `formatters`, `tool_output`, `workflows`, `goals`, `teams`, `loops`, `messaging`, `peers`, `remote`, `runners`, `channels`, `account`, `share`, `telemetry`, `budgets`, `policy`, `profiles`, `default_profile`, `shell`, `attachments`, `background`, `tools`, `git`, `ide`, `autofix`, `network`, `storage`, `compat`, `services`, `features`, `experimental` (deprecated alias of `features`) and `autoupdate` (`installation-upgrade`). Collection keys SHALL use plural names, and entries SHALL use `disabled: true` to turn off an inherited entry. #### Scenario: Disabling an inherited MCP server - **WHEN** the global config defines `mcp.github` and the project config sets `mcp.github.disabled = true` @@ -10,3 +10,12 @@ #### Scenario: Invalid output style name - **WHEN** `cyber.jsonc` sets `output_style = "Bad Style"` - **THEN** validation fails with `ConfigInvalidError` naming `output_style` + +#### Scenario: Layered instruction budgets preserve instruction lists +- **WHEN** a global configuration sets both instruction limits, a project overrides the total, and a selected profile or command-line override changes the file limit +- **THEN** normal member precedence SHALL determine the effective limits and `instructions` arrays SHALL retain their existing concatenation semantics +- **AND** null, noninteger, nonpositive, unknown or incompatible limit values SHALL fail validation before the configuration is usable + +#### Scenario: Project budgets require workspace trust +- **WHEN** an untrusted project defines `instruction_limits`, directly or inside a profile +- **THEN** that definition SHALL remain withheld under the existing workspace-trust digest until approved, without changing trusted global limits diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index 40fdac61e..5c439623b 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -10,3 +10,28 @@ #### Scenario: Inspection of an untrusted source - **WHEN** an owner inspects context while project sources are withheld by trust policy - **THEN** inspection reports their withheld state without loading their content or treating their estimated bytes as active context + +## MODIFIED Requirements + +### Requirement: Path-scoped rules and overrides +(P1) The system SHALL load rule files from `.cyber/rules/*.md` and, unless `compat.claude` is false, `.claude/rules/*.md` in each config directory. A rule file without frontmatter `paths` SHALL join `core/instructions` in the baseline. A rule file with `paths: [globs]` (relative to its project root) SHALL be attached through the nested-instruction mechanism the first time per Context Epoch that a matching file is read, edited or patched, labelled with its path. An `AGENTS.override.md` in a directory SHALL take the `AGENTS.md` position and replace that directory's `AGENTS.md`, `CLAUDE.md` and `CYBER.md` instead of being appended; that directory's `CONTEXT.md` still loads. Each instruction or rule file, including `CYBER.md`, SHALL be truncated at `instruction_limits.max_file_bytes` (default 32768) with a notice, and `cyber doctor` SHALL warn about truncated files. The baseline instruction text as a whole, imports included, SHALL be bounded by `instruction_limits.max_total_bytes` (default 131072 bytes): the file that crosses the bound SHALL be cut with a notice, later files and imports SHALL be left out with a warning. + +#### Scenario: Migration rules only for migrations +- **WHEN** `.cyber/rules/migrations.md` has `paths: ["db/migrations/**"]` and the model edits `db/migrations/0042.sql` +- **THEN** the edit result includes the rule file in a `` block, once for the epoch, and unrelated edits never load it + +#### Scenario: Override file replaces +- **WHEN** `/repo/packages/api/` contains both `AGENTS.md` and `AGENTS.override.md` +- **THEN** only `AGENTS.override.md` is loaded for that directory + +#### Scenario: Override file replaces CYBER.md +- **WHEN** `/repo/packages/api/` contains `AGENTS.override.md`, `CYBER.md` and `CONTEXT.md` +- **THEN** `AGENTS.override.md` and then `CONTEXT.md` are loaded for that directory, and `CYBER.md` is not + +#### Scenario: Oversized CYBER.md +- **WHEN** `/repo/CYBER.md` is larger than `instruction_limits.max_file_bytes` +- **THEN** its content is cut at that size with a truncation notice naming the file, and `cyber doctor` warns about it + +#### Scenario: Many imports +- **WHEN** `/repo/CYBER.md` imports enough files to exceed the total instruction budget +- **THEN** the baseline stops at the budget with a truncation notice, and a warning names each file left out diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index b5916c117..3e7414a24 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1535,3 +1535,44 @@ managed metadata process-death selection passes locally after this correction (two entries, including the inert child entry and parent exercising both cuts). Formatting and diff checks pass. Linker unwind and native C deprecation warnings were observed without assigning origin. Linux execution remains required. + +Instruction budget continuation (2026-11-10): the approved additive +`instruction_limits` object now validates positive integer file/total limits +(defaults 32768/131072, file <= total), rejects nonobjects/unknown members, and +uses normal member layering, profiles and CLI overrides. Project definitions +(including profile members) use the existing sensitive-definition trust digest; +held definitions cannot change trusted global limits. `instructions` array +concatenation is preserved. No managed-policy namespace is added or widened. + +The typed limits flow through ToolHost, BuiltinHost and AppHost into baseline and +nested loading and doctor. Nested text honors a configured total below its 32 KiB +ceiling. Configuration failure withholds instruction reads without hiding date +or environment; first-epoch initialization keeps input retryable with no model +call, and repair resumes it. Diagnostic names now identify the actual setting. +The maximum-integer read limit saturates its lookahead instead of overflowing. + +Review/regressions caught and resolved a missing AppHost forwarding method, +Serde accepting an empty array for a defaulted struct, missing project trust +gating and maximum-limit overflow. The real-App regression inspects actual +provider baseline and nested text, including UTF-8/file/total cuts. Verification: +88 configuration tests; 444 server library tests with one explicit subprocess +fixture ignored; 20 context/compaction tests; 9 instruction App tests; the AppHost +forwarding check; 2 host configuration tests; and the doctor regression pass. The +focused maximum-limit read test passes after the arithmetic fix. All-target +Clippy for core/server/tools/App/CLI passes with warnings denied after moving the +doctor test module to the end of the file; no allowance is added. Native C and +linker unwind warnings were observed without assigning origin. + +Formatting, diff checks, SDK generation consistency and workflow YAML/independent +step/log checks pass. CI-pinned OpenSpec 1.13.2 validates 100/100 items; lint reports +0 errors/25 warnings. No HTTP/CLI envelope or generated artifact changed; SDK +runtime tests and full workspace tests were not rerun for this continuation. The +native workflow now selects and retains these instruction/context regressions. +Current-source native/platform acceptance remains required. + +Focused code-review-skill review finds no unresolved finding in this settings +wiring. Full rendered assembly accounting (labels/notices/omitted inventories), +trusted path-rule discovery/globs, successful-operation/Epoch/digest-bound +reminders, changed/revoked definitions, effective context inspection and the TUI +flow remain required. Task 4.1 and all other parent tasks stay open; this does not +accept rules/context, the recovery branch or issue #93. From 1922fdaeb0dacb14e2d57c1840242cbf394544c8 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 08:25:44 -0400 Subject: [PATCH 75/87] Bound complete instruction rendering and exact reminder delivery --- .github/workflows/non-git-snapshot-proof.yml | 3 + crates/cyber-app/tests/instructions.rs | 103 ++++- crates/cyber-cli/src/commands/doctor.rs | 2 +- crates/cyber-server/src/runtime/drain.rs | 80 ++-- .../cyber-server/src/runtime/instructions.rs | 355 ++++++++++++------ .../src/runtime/instructions/assembly.rs | 191 ++++++++++ docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 4 +- .../complete-daily-coding-surfaces/design.md | 2 +- .../specs/system-context/spec.md | 6 + .../complete-daily-coding-surfaces/tasks.md | 45 +++ 11 files changed, 630 insertions(+), 163 deletions(-) create mode 100644 crates/cyber-server/src/runtime/instructions/assembly.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 0ac1b6e70..f7403d08a 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -32,6 +32,7 @@ on: - crates/cyber-tools/tests/instruction_config.rs - crates/cyber-server/src/runtime/context.rs - crates/cyber-server/src/runtime/instructions.rs + - crates/cyber-server/src/runtime/instructions/** - crates/cyber-server/src/runtime/host.rs - crates/cyber-server/tests/context_compaction.rs - crates/cyber-server/tests/support/mod.rs @@ -194,6 +195,8 @@ jobs: mkdir -p "$TMPDIR" cargo test --locked -p cyber-core --test config instruction_limits -- --nocapture 2>&1 | tee instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::context:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-server --lib runtime::instructions:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-server --lib runtime::drain::skill_reminder_tests:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --test context_compaction -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-app --features native-git-acceptance --test instructions -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-tools --test instruction_config -- --nocapture 2>&1 | tee -a instruction-limits-proof.log diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index e5663ea67..da86cd80e 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -44,29 +44,95 @@ fn reply(body: &Value) -> Value { #[tokio::test] async fn configured_instruction_limits_bound_baseline_and_nested_provider_text() { let f = Fixture::start_with( - json!({"instruction_limits":{"max_file_bytes":16,"max_total_bytes":24}}), + json!({"instruction_limits":{"max_file_bytes":3000,"max_total_bytes":4096}}), ) .await; - f.write("AGENTS.md", &"λ".repeat(20)); - f.write("CYBER.md", &"Ж".repeat(20)); - f.write("web/AGENTS.md", &"Ψ".repeat(20)); - f.write("web/CYBER.md", &"Ω".repeat(20)); + f.write("AGENTS.md", &"λ".repeat(5000)); + f.write("web/AGENTS.md", &"Ψ".repeat(5000)); + f.write("web/CYBER.md", &"Ω".repeat(5000)); f.write("web/src/app.ts", "export const app = true;"); let session = f.session(&f.repo).await; f.prompt(&session, &f.repo, "read the app first", 2).await; let body = f.last_request(); - let baseline = system_prompt(&body); - assert!(baseline.contains(&"λ".repeat(8)), "{baseline}"); - assert!(!baseline.contains(&"λ".repeat(9)), "{baseline}"); - assert!(baseline.contains(&"Ж".repeat(4)), "{baseline}"); - assert!(!baseline.contains(&"Ж".repeat(5)), "{baseline}"); - assert!(baseline.contains("instruction_limits.max_file_bytes (16 bytes)")); + let state = f.app.runtime.state(&session).await.unwrap(); + let epoch = state.epoch.unwrap(); + let instructions = &epoch.snapshot["core/instructions"]; + assert!(instructions.len() <= 4096); + assert!(instructions.contains(&"λ".repeat(1500))); + assert!(!instructions.contains(&"λ".repeat(1501))); + assert!(system_prompt(&body).contains(instructions)); let nested = last_tool_result(&body); - assert!(nested.contains(&"Ψ".repeat(8)), "{nested}"); - assert!(!nested.contains(&"Ψ".repeat(9)), "{nested}"); - assert!(nested.contains(&"Ω".repeat(4)), "{nested}"); - assert!(!nested.contains(&"Ω".repeat(5)), "{nested}"); - assert!(nested.contains("total budget")); + let start = nested.find("\n\n").unwrap(); + let reminder = &nested[start..]; + assert!(reminder.len() <= 4096, "{} bytes", reminder.len()); + assert!(reminder.ends_with("")); + assert!(reminder.contains("Ψ")); + assert!(!reminder.contains(&"Ψ".repeat(1501))); + assert!(!reminder.contains(&"Ω".repeat(1501))); + assert!(reminder.contains("total budget") || reminder.contains("instruction budget")); + + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn bounded_lists_record_only_labelled_files_and_retry_unlisted_imports() { + let f = Fixture::start_with( + json!({"instruction_limits":{"max_file_bytes":512,"max_total_bytes":512}}), + ) + .await; + let refs: Vec<_> = (0..20).map(|i| format!("@r/{i:02}.md")).collect(); + f.write("web/CYBER.md", &refs.join("\n")); + for i in 0..20 { + f.write(&format!("web/r/{i:02}.md"), "x"); + } + f.write("web/src/app.ts", "export const app = true;"); + let session = f.session(&f.repo).await; + f.prompt(&session, &f.repo, "read the app first", 2).await; + let first = f + .app + .runtime + .state(&session) + .await + .unwrap() + .epoch + .unwrap() + .reminded_instructions; + let output = last_tool_result(&f.last_request()); + let reminder = &output[output.find("\n\n").unwrap()..]; + assert!(reminder.len() <= 512); + assert!(!first.is_empty() && first.len() < 21); + for id in &first { + assert!( + reminder.contains(id), + "unlabelled identity {id}: {reminder}" + ); + } + f.prompt(&session, &f.repo, "read the app again", 2).await; + let second = f + .app + .runtime + .state(&session) + .await + .unwrap() + .epoch + .unwrap() + .reminded_instructions; + let output = last_tool_result(&f.last_request()); + let reminder = &output[output.find("\n\n").unwrap()..]; + assert!(reminder.len() <= 512); + assert!(second.len() > first.len()); + for id in second.difference(&first) { + assert!( + reminder.contains(id), + "unlabelled identity {id}: {reminder}" + ); + } + for id in &first { + assert!( + !reminder.contains(id), + "duplicate identity {id}: {reminder}" + ); + } f.app.runtime.shutdown().await; } @@ -385,7 +451,10 @@ async fn two_full_size_nested_files_still_give_a_successful_read() { &result[..result.len().min(400)] ); assert!(result.contains("export const app")); - assert!(result.contains(&"a".repeat(32 * 1024))); + assert!(result.contains(&"a".repeat(1024))); + assert!(!result.contains(&"a".repeat(32 * 1024))); + let reminder = &result[result.find("\n\n").unwrap()..]; + assert!(reminder.len() <= 32 * 1024); assert!(result.contains(&format!("- {}", cyber.display()))); assert!(!result.contains("cccc")); f.prompt(&session, &repo, "read the app again", 2).await; diff --git a/crates/cyber-cli/src/commands/doctor.rs b/crates/cyber-cli/src/commands/doctor.rs index a3bd558e0..10aa8931b 100644 --- a/crates/cyber-cli/src/commands/doctor.rs +++ b/crates/cyber-cli/src/commands/doctor.rs @@ -531,7 +531,7 @@ mod instruction_tests { std::fs::write(ctx.paths.config.join("AGENTS.md"), "λ".repeat(40)).unwrap(); std::fs::write( ctx.paths.config.join("cyber.json"), - r#"{"instruction_limits":{"max_file_bytes":16,"max_total_bytes":24}}"#, + r#"{"instruction_limits":{"max_file_bytes":16,"max_total_bytes":1024}}"#, ) .unwrap(); let resolved = cyber_core::config::load(&cyber_core::config::LoadRequest { diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index c45585306..914401819 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -2084,7 +2084,8 @@ impl Inner { /// The instruction files each successful read in `targets` attaches (`system-context` /// → Nested rule files on read), skipping the baseline, files already attached this - /// epoch and files an earlier target attaches. The files are read on a blocking thread + /// epoch. In-group deduplication happens only after actual reminder finalization. + /// The files are read on a blocking thread /// before the caller takes the Session lock, so a slow file never stalls the Session. pub(super) async fn nested_instructions( &self, @@ -2096,7 +2097,7 @@ impl Inner { if targets.iter().all(Option::is_none) { return empty(); } - let mut attached = Reminded::of(&*handle.state.lock().await).instructions; + let attached = Reminded::of(&*handle.state.lock().await).instructions; let global = self.options.global_config_dir.clone(); let load = move || { let mut loaded = Vec::with_capacity(targets.len()); @@ -2110,7 +2111,6 @@ impl Inner { &attached, ) }); - attached.extend(nested.attached()); loaded.push(nested); } loaded @@ -2408,24 +2408,34 @@ pub(super) fn settlement_with_reminders( let (names, skill_reminder) = skill_reminder(skills, &seen.skills); let skill_reminder = skill_reminder.unwrap_or_default(); let nested = if failed { None } else { Some(nested) }; - let (output, instructions) = match with_reminders(output, nested, &skill_reminder, &finalize) { + let (output, instructions) = match with_reminders( + output, + nested, + &seen.instructions, + &skill_reminder, + &finalize, + ) { Ok(done) => done, Err(error) => return (settlement(call_id, def, ToolOutcome::Crashed(error)), None), }; - let attached = match (&instructions, nested) { - (Some(_), Some(nested)) => nested.attached(), - _ => Vec::new(), - }; + let attached = instructions.as_ref().map_or_else(Vec::new, |delivery| { + delivery + .files + .iter() + .map(|file| file.canonical.display().to_string()) + .collect::>() + }); seen.skills.extend(names.iter().cloned()); seen.instructions.extend(attached.iter().cloned()); - let loaded = - instructions - .zip(nested) - .map(|(text, nested)| super::context::LoadedInstructions { - source: "core/instructions".into(), - text, - files: nested.paths(), - }); + let loaded = instructions.map(|delivery| super::context::LoadedInstructions { + source: "core/instructions".into(), + text: delivery.text, + files: delivery + .files + .iter() + .map(|file| file.path.display().to_string()) + .collect(), + }); let settled = event( TOOL_SETTLED, &ToolSettled { @@ -2456,15 +2466,21 @@ pub(super) fn settlement_with_reminders( fn with_reminders( output: String, nested: Option<&super::instructions::Instructions>, + seen: &std::collections::BTreeSet, skill: &str, finalize: &impl Fn(String, &str) -> Result, -) -> Result<(String, Option), String> { +) -> Result<(String, Option), String> { let candidates = nested .into_iter() - .flat_map(|nested| [nested.reminder(), nested.pointer()]) + .flat_map(|nested| { + [ + nested.reminder_delivery(seen), + nested.pointer_delivery(seen), + ] + }) .flatten(); for instructions in candidates { - if let Ok(done) = finalize(output.clone(), &format!("{instructions}{skill}")) { + if let Ok(done) = finalize(output.clone(), &format!("{}{skill}", instructions.text)) { return Ok((done, Some(instructions))); } } @@ -2818,7 +2834,7 @@ mod skill_reminder_tests { (serde_json::from_value(event.data).unwrap(), loaded) } - /// Regression: two nested files near `instructions.max_bytes` made a reminder larger + /// Regression: two nested files near the file limit made a reminder larger /// than the tool output budget, the read settled as `Tool crashed`, and nothing was /// recorded, so every later read in that directory crashed too. #[test] @@ -2831,10 +2847,14 @@ mod skill_reminder_tests { let (payload, loaded) = settle(&nested, &mut seen, default_budget); assert_eq!(payload.status, CallStatus::Ok, "{}", payload.output); assert!(payload.output.starts_with("file text")); - assert!(payload.output.contains(&"a".repeat(32 * 1024))); + assert!(payload.output.contains(&"a".repeat(1024))); + assert!(!payload.output.contains(&"a".repeat(32 * 1024))); assert!(!payload.output.contains("ccc")); assert!(payload.output.contains("exceed the instruction budget")); - assert!(payload.output.len() <= 51_200 + "file text".len()); + assert!( + payload.output.len() + <= super::super::instructions::NESTED_MAX_TOTAL_BYTES + "file text".len() + ); assert_eq!(payload.instruction_reminders.len(), 2); assert_eq!(seen.instructions.len(), 2); assert_eq!(loaded.unwrap().files.len(), 2); @@ -2859,6 +2879,22 @@ mod skill_reminder_tests { ); } + #[test] + fn only_successfully_finalized_reminders_reserve_files_in_a_group() { + let (_temp, nested) = nested(&[("CYBER.md", "rule".into())]); + let mut seen = Reminded::default(); + let (_, first) = settle(&nested, &mut seen, |_, _| Err("no room".into())); + assert!(first.is_none()); + assert!(seen.instructions.is_empty()); + let (second, loaded) = settle(&nested, &mut seen, default_budget); + assert_eq!(second.instruction_reminders.len(), 1); + assert_eq!(loaded.unwrap().files.len(), 1); + let (third, loaded) = settle(&nested, &mut seen, default_budget); + assert!(third.instruction_reminders.is_empty()); + assert!(loaded.is_none()); + assert_eq!(third.output, "file text"); + } + /// Regression: nested instruction files were loaded for every read, denied and failed /// ones included, before the outcome was checked. #[test] diff --git a/crates/cyber-server/src/runtime/instructions.rs b/crates/cyber-server/src/runtime/instructions.rs index 234bb6fda..5e9e13d34 100644 --- a/crates/cyber-server/src/runtime/instructions.rs +++ b/crates/cyber-server/src/runtime/instructions.rs @@ -16,6 +16,9 @@ use std::sync::Arc; use cyber_core::env::EnvSource; +mod assembly; +pub(crate) use assembly::Reminder as InstructionReminder; + /// Files read from each directory, in load order. `CLAUDE.md` is read only where the /// directory has no `AGENTS.md`. `CYBER.md`, Cyber Code's own instruction file, comes /// last so Cyber-specific guidance wins ties within a directory. @@ -121,83 +124,112 @@ pub struct InstructionFile { pub truncated: bool, } -/// Rendered blocks, the files behind them in load order, and non-fatal problems. +/// Bounded instruction bodies, the files behind them, and non-fatal problems. #[derive(Debug, Default)] pub struct Instructions { - pub blocks: Vec, + blocks: Vec, pub files: Vec, /// Files that apply but were left out because the total budget was spent. pub omitted: Vec, pub warnings: Vec, + max_file_bytes: usize, + max_total_bytes: usize, } impl Instructions { + fn new(max_file_bytes: usize, max_total_bytes: usize) -> Self { + Self { + max_file_bytes, + max_total_bytes, + ..Default::default() + } + } + pub fn text(&self) -> Option { - (!self.blocks.is_empty()).then(|| self.blocks.join("\n\n")) + let (text, _) = assembly::blocks( + self.blocks.iter(), + self.max_file_bytes, + self.max_total_bytes, + ); + (!text.is_empty()).then_some(text) } - /// Whether a nested read has anything to attach. pub fn is_empty(&self) -> bool { self.blocks.is_empty() && self.omitted.is_empty() } - /// The text appended to a tool result for nested instructions, if any: the files' - /// text, then the paths of the files the budget left out. pub fn reminder(&self) -> Option { - let mut parts: Vec = self.text().into_iter().collect(); - if !self.omitted.is_empty() { - parts.push(pointer_text( - "These instruction files also apply here but exceed the instruction budget", - &self.omitted, - )); - } - (!parts.is_empty()).then(|| reminder_block(&parts.join("\n\n"))) + self.reminder_delivery(&BTreeSet::new()) + .map(|delivery| delivery.text) } - /// A short reminder naming every file, for when [`Self::reminder`] does not fit the - /// tool output budget. pub fn pointer(&self) -> Option { - let all: Vec = self.files.iter().chain(&self.omitted).cloned().collect(); - (!all.is_empty()).then(|| { - reminder_block(&pointer_text( - "These instruction files apply here but exceed the tool output budget", - &all, - )) - }) + self.pointer_delivery(&BTreeSet::new()) + .map(|delivery| delivery.text) } - /// Canonical paths of every file a reminder covers (attached or named). - pub fn attached(&self) -> Vec { - self.files - .iter() - .chain(&self.omitted) - .map(|file| file.canonical.display().to_string()) - .collect() + fn content_budget(&self) -> usize { + self.max_total_bytes + .saturating_sub(assembly::OPEN.len() + assembly::CLOSE.len()) } - /// The paths of the files a reminder covers, as discovered. - pub fn paths(&self) -> Vec { - self.files - .iter() - .chain(&self.omitted) - .map(|file| file.path.display().to_string()) - .collect() + pub(crate) fn reminder_delivery(&self, seen: &BTreeSet) -> Option { + let eligible = + |file: &&InstructionFile| !seen.contains(&file.canonical.display().to_string()); + // Reserve complete labels first when some files could not enter the body. + // A bounded prefix is truthful: unlisted files remain eligible next time. + let pointers = (!self.omitted.is_empty()) + .then(|| { + assembly::pointers( + "Instruction files for this path that may exceed the instruction budget", + self.files.iter().chain(&self.omitted).filter(eligible), + self.content_budget(), + ) + }) + .flatten(); + let reserved = pointers + .as_ref() + .map_or(0, |pointer| pointer.text.len().saturating_add(2)); + let (mut text, mut files) = assembly::blocks( + self.blocks + .iter() + .filter(|block| !seen.contains(&block.file.canonical.display().to_string())), + self.max_file_bytes, + self.content_budget().saturating_sub(reserved), + ); + if let Some(pointer) = pointers { + if !text.is_empty() { + text.push_str("\n\n"); + } + text.push_str(&pointer.text); + files.extend(pointer.files); + } + assembly::finish(text, files) } -} -fn reminder_block(text: &str) -> String { - format!("\n\n\n{text}\n") -} + pub(crate) fn pointer_delivery(&self, seen: &BTreeSet) -> Option { + let pointer = assembly::pointers( + "These instruction files apply here but exceed the tool output budget", + self.files + .iter() + .chain(&self.omitted) + .filter(|file| !seen.contains(&file.canonical.display().to_string())), + self.content_budget(), + )?; + assembly::finish(pointer.text, pointer.files) + } -fn pointer_text(reason: &str, files: &[InstructionFile]) -> String { - let list: Vec = files - .iter() - .map(|file| format!("- {}", file.path.display())) - .collect(); - format!( - "{reason}; read them with the read tool:\n{}", - list.join("\n") - ) + /// Canonical identities actually labelled by the default bounded delivery. + pub fn attached(&self) -> Vec { + self.reminder_delivery(&BTreeSet::new()) + .map_or_else(Vec::new, |delivery| { + delivery + .files + .iter() + .map(|file| file.canonical.display().to_string()) + .collect() + }) + } } /// The candidate files of one directory, in load order. @@ -265,11 +297,18 @@ pub fn load_nested( if !settings.project || dirs.is_empty() { return Instructions::default(); } - let mut loader = Loader::new(settings, settings.max_total_bytes); - let _ = fill_baseline(&mut loader, global_dir, &location); - loader.out = Instructions::default(); - loader.remaining = NESTED_MAX_TOTAL_BYTES.min(settings.max_total_bytes); - loader.seen.extend(attached.iter().map(PathBuf::from)); + let mut baseline = Loader::new(settings, settings.max_total_bytes); + let _ = fill_baseline(&mut baseline, global_dir, &location); + let budget = NESTED_MAX_TOTAL_BYTES.min(settings.max_total_bytes); + let mut loader = Loader::new(settings, budget); + loader.delivered = baseline + .out + .files + .iter() + .map(|file| file.canonical.clone()) + .collect(); + loader.delivered.extend(attached.iter().map(PathBuf::from)); + loader.remaining = budget.saturating_sub(assembly::OPEN.len() + assembly::CLOSE.len()); let root = canonical(&cyber_core::config::project_root(&location)); for file in dirs.iter().flat_map(|dir| directory_files(dir, settings)) { if let Err(error) = loader.load(&file, Some(&root)) { @@ -327,6 +366,8 @@ fn fill_baseline(loader: &mut Loader<'_>, global_dir: &Path, location: &Path) -> struct Loader<'a> { settings: &'a InstructionSettings, seen: BTreeSet, + delivered: BTreeSet, + discovery_remaining: usize, /// Canonical paths of the files whose imports are being expanded. stack: Vec, /// Bytes of instruction text still allowed. @@ -339,9 +380,11 @@ impl<'a> Loader<'a> { Self { settings, seen: BTreeSet::new(), + delivered: BTreeSet::new(), + discovery_remaining: settings.max_total_bytes, stack: Vec::new(), remaining: budget, - out: Instructions::default(), + out: Instructions::new(settings.max_bytes, budget), } } @@ -380,75 +423,98 @@ impl<'a> Loader<'a> { if !self.seen.insert(key.clone()) { return Ok(()); } + if self.delivered.contains(&key) { + return self.discover_imports(path, key, confine); + } let file = InstructionFile { path: path.to_path_buf(), canonical: key.clone(), truncated: false, }; - if self.remaining == 0 { - self.out.warnings.push(format!( - "{} was not loaded: instruction files exceed their total budget", - path.display() - )); - self.out.omitted.push(file); - return Ok(()); + let separator = if self.out.blocks.is_empty() { 0 } else { 2 }; + let label = format!("{INSTRUCTIONS_FROM}{}\n", path.display()); + let available = self.remaining.saturating_sub(separator); + if available < label.len() { + return self.omit(file); } - let limit = self.remaining.min(self.settings.max_bytes); + let limit = self + .settings + .max_bytes + .min(available - label.len()) + .min(self.discovery_remaining); let read = read_bounded(path, limit)?; - let body = self.render(path, read, limit); - self.remaining -= body.kept; - self.out.files.push(InstructionFile { - truncated: body.omitted > 0, + self.discovery_remaining -= read.text.len(); + let Some(rendered) = assembly::block(path, &read, self.settings.max_bytes, available) + else { + return self.omit(file); + }; + self.remaining -= rendered.text.len() + separator; + let file = InstructionFile { + truncated: rendered.body.omitted > 0, ..file + }; + if file.truncated { + self.out + .warnings + .push(if rendered.body.limit == self.settings.max_bytes { + format!( + "{} exceeds instruction_limits.max_file_bytes ({} bytes) and was truncated", + path.display(), + self.settings.max_bytes + ) + } else { + format!( + "{} was truncated: instruction files exceed their total budget", + path.display() + ) + }); + } + let references = imports(rendered.body.complete_lines()); + self.out.files.push(file.clone()); + self.out.blocks.push(assembly::Block { + file, + body: rendered.body, }); self.stack.push(key); - for reference in imports(&body.text) { + for reference in references { self.import(path, &reference, confine); } self.stack.pop(); Ok(()) } - /// The block for `read`, with a truncation notice and warning when it was cut. - fn render(&mut self, path: &Path, read: Bounded, limit: usize) -> Rendered { - let (kept, omitted) = (read.text.len(), read.omitted); - let mut block = read.text.trim_end().to_owned(); - if omitted > 0 { - let max = self.settings.max_bytes; - let (notice, warning) = if limit == max { - ( - format!( - "[Truncated: {} exceeds instruction_limits.max_file_bytes ({max} bytes); {omitted} bytes omitted]", - path.display() - ), - format!( - "{} exceeds instruction_limits.max_file_bytes ({max} bytes) and was truncated", - path.display() - ), - ) - } else { - ( - format!( - "[Truncated: instruction files exceed their total budget; {omitted} bytes of {} omitted]", - path.display() - ), - format!( - "{} was truncated: instruction files exceed their total budget", - path.display() - ), - ) - }; - block = format!("{}\n{notice}", read.text); - self.out.warnings.push(warning); + /// Revisit authorized import declarations without redelivering an already labelled + /// parent. Otherwise a bounded pointer prefix would strand its unlisted children. + fn discover_imports( + &mut self, + path: &Path, + key: PathBuf, + confine: Option<&Path>, + ) -> Result<(), String> { + if self.discovery_remaining == 0 { + self.out.warnings.push(format!( + "{}: import discovery budget exhausted", + path.display() + )); + return Ok(()); } - self.out - .blocks - .push(format!("{INSTRUCTIONS_FROM}{}\n{block}", path.display())); - Rendered { - text: read.text, - kept, - omitted, + let read = read_bounded(path, self.settings.max_bytes.min(self.discovery_remaining))?; + self.discovery_remaining -= read.text.len(); + self.stack.push(key); + for reference in imports(read.complete_lines()) { + self.import(path, &reference, confine); } + self.stack.pop(); + Ok(()) + } + + fn omit(&mut self, file: InstructionFile) -> Result<(), String> { + self.out.warnings.push(format!( + "{} was not loaded: instruction files exceed their total budget", + file.path.display() + )); + self.out.omitted.push(file); + Ok(()) } fn import(&mut self, from: &Path, reference: &str, confine: Option<&Path>) { @@ -501,16 +567,25 @@ impl<'a> Loader<'a> { } /// A file's text read up to a byte limit, and how many bytes past it were left unread. +#[derive(Debug, Clone)] struct Bounded { text: String, omitted: usize, + limit: usize, } -/// The kept text of a rendered file (imports are parsed from it) and its size. -struct Rendered { - text: String, - kept: usize, - omitted: usize, +impl Bounded { + /// A cut final line is not an import declaration: its truncated path could name + /// a different existing file. Full final lines without a newline remain valid. + fn complete_lines(&self) -> &str { + if self.omitted > 0 && !self.text.ends_with('\n') { + self.text + .rsplit_once('\n') + .map_or("", |(complete, _)| complete) + } else { + &self.text + } + } } /// Read at most `limit` bytes of `path`, cut back to a character boundary. A path that @@ -551,7 +626,11 @@ fn read_bounded(path: &Path, limit: usize) -> Result { } else { 0 }; - Ok(Bounded { text, omitted }) + Ok(Bounded { + text, + omitted, + limit, + }) } /// `@path` references on their own line, outside code fences. @@ -1008,11 +1087,9 @@ mod tests { settings.max_total_bytes = 350; let loaded = repo.baseline("", &settings); let text = loaded.text().unwrap(); - assert_eq!(loaded.files.len(), 4, "{:?}", loaded.files); - assert!(loaded.files[3].truncated); - assert!(text.contains("instruction files exceed their total budget")); - assert_eq!(loaded.omitted.len(), 7); - assert!(text.matches('y').count() <= 350); + assert!(text.len() <= 350, "{} bytes", text.len()); + assert!(!loaded.omitted.is_empty()); + assert!(text.contains("CYBER.md")); assert!( loaded .warnings @@ -1033,7 +1110,7 @@ mod tests { assert_eq!(order(&repo, &nested), ["web/AGENTS.md"]); assert_eq!(nested.omitted.len(), 1); let reminder = nested.reminder().unwrap(); - assert!(reminder.len() < DEFAULT_MAX_BYTES + 1024); + assert!(reminder.len() <= NESTED_MAX_TOTAL_BYTES); assert!(reminder.contains(&format!("- {}", cyber.display()))); assert_eq!(nested.attached().len(), 2); let pointer = nested.pointer().unwrap(); @@ -1042,4 +1119,44 @@ mod tests { "{pointer}" ); } + + #[test] + fn a_cut_import_line_never_loads_the_file_named_by_its_prefix() { + let repo = Repo::new(); + repo.write("CYBER.md", "@secrets.md"); + repo.write("secret", "unintended file"); + let mut settings = settings(); + settings.max_bytes = 7; + let loaded = repo.baseline("", &settings); + assert_eq!(order(&repo, &loaded), ["CYBER.md"]); + } + + #[test] + fn unlisted_imports_remain_discoverable_after_their_parent_was_labelled() { + let repo = Repo::new(); + let refs: Vec<_> = (0..20).map(|i| format!("@r/{i:02}.md")).collect(); + repo.write("web/CYBER.md", &refs.join("\n")); + for i in 0..20 { + repo.write(&format!("web/r/{i:02}.md"), "x"); + } + let target = repo.write("web/app.ts", ""); + let mut settings = settings(); + settings.max_bytes = 512; + settings.max_total_bytes = 512; + let mut attached = BTreeSet::new(); + for _ in 0..22 { + let nested = load_nested(&repo.global, &repo.root, &target, &settings, &attached); + let Some(delivery) = nested.reminder_delivery(&attached) else { + break; + }; + assert!(delivery.text.len() <= 512); + let before = attached.len(); + for file in delivery.files { + assert!(delivery.text.contains(&file.path.display().to_string())); + attached.insert(file.canonical.display().to_string()); + } + assert!(attached.len() > before); + } + assert_eq!(attached.len(), 21, "{attached:?}"); + } } diff --git a/crates/cyber-server/src/runtime/instructions/assembly.rs b/crates/cyber-server/src/runtime/instructions/assembly.rs new file mode 100644 index 000000000..09f90a034 --- /dev/null +++ b/crates/cyber-server/src/runtime/instructions/assembly.rs @@ -0,0 +1,191 @@ +//! Render complete labels/notices within a byte budget; never infer delivery from text. +use super::{Bounded, INSTRUCTIONS_FROM, InstructionFile}; +use std::collections::BTreeSet; +use std::path::Path; + +pub(super) const OPEN: &str = "\n\n\n"; +pub(super) const CLOSE: &str = "\n"; + +#[derive(Debug)] +pub(super) struct Block { + pub file: InstructionFile, + pub body: Bounded, +} + +pub(super) struct Rendered { + pub text: String, + pub body: Bounded, +} + +/// The exact files whose complete labels occur in this particular delivery. +#[derive(Debug)] +pub(crate) struct Reminder { + pub text: String, + pub files: Vec, +} + +pub(super) fn block( + path: &Path, + body: &Bounded, + max_file: usize, + budget: usize, +) -> Option { + let label = format!("{INSTRUCTIONS_FROM}{}\n", path.display()); + let mut body = body.clone(); + loop { + let notice = notice(path, body.omitted, max_file, body.limit == max_file); + let room = budget.checked_sub(label.len().saturating_add(notice.len()))?; + if body.text.len() <= room { + let text = if body.omitted == 0 { + body.text.trim_end() + } else { + &body.text + }; + return Some(Rendered { + text: format!("{label}{text}{notice}"), + body, + }); + } + let mut end = room; + while !body.text.is_char_boundary(end) { + end -= 1; + } + body.omitted = body.omitted.saturating_add(body.text.len() - end); + body.text.truncate(end); + body.limit = body.limit.min(room); + } +} + +fn notice(path: &Path, omitted: usize, max: usize, per_file: bool) -> String { + if omitted == 0 { + return String::new(); + } + if per_file { + format!( + "\n[Truncated: {} exceeds instruction_limits.max_file_bytes ({max} bytes); {omitted} bytes omitted]", + path.display() + ) + } else { + format!( + "\n[Truncated: instruction files exceed their total budget; {omitted} bytes of {} omitted]", + path.display() + ) + } +} + +pub(super) fn blocks<'a>( + blocks: impl Iterator, + max_file: usize, + budget: usize, +) -> (String, Vec) { + let mut text = String::new(); + let mut files = Vec::new(); + for source in blocks { + let separator = if text.is_empty() { "" } else { "\n\n" }; + let remaining = budget.saturating_sub(text.len().saturating_add(separator.len())); + if let Some(rendered) = block(&source.file.path, &source.body, max_file, remaining) { + text.push_str(separator); + text.push_str(&rendered.text); + let mut file = source.file.clone(); + file.truncated |= rendered.body.omitted > 0; + files.push(file); + } + } + (text, files) +} + +/// Include only whole path labels. Unlisted paths confer no delivery authority. +pub(super) fn pointers<'a>( + reason: &str, + files: impl Iterator, + budget: usize, +) -> Option { + let mut text = format!("{reason}; read them with the read tool:"); + let mut included = Vec::new(); + for file in files { + let line = format!("\n- {}", file.path.display()); + if text.len().saturating_add(line.len()) <= budget { + text.push_str(&line); + included.push(file.clone()); + } + } + (!included.is_empty()).then_some(Reminder { + text, + files: included, + }) +} + +pub(super) fn finish(text: String, files: Vec) -> Option { + if files.is_empty() { + return None; + } + let mut seen = BTreeSet::new(); + let files = files + .into_iter() + .filter(|file| seen.insert(file.canonical.clone())) + .collect(); + Some(Reminder { + text: format!("{OPEN}{text}{CLOSE}"), + files, + }) +} + +#[cfg(test)] +mod tests { + use super::super::Instructions; + use super::*; + + #[test] + fn labels_notices_and_utf8_body_fit_every_small_budget() { + let path = Path::new("/repo/日本語/CYBER.md"); + let body = Bounded { + text: "λ".repeat(800), + omitted: 500, + limit: 1600, + }; + for budget in 0..1024 { + if let Some(rendered) = block(path, &body, 1600, budget) { + assert!(rendered.text.len() <= budget); + assert!( + rendered + .text + .starts_with(&format!("{INSTRUCTIONS_FROM}{}\n", path.display())) + ); + assert!(rendered.text.ends_with("omitted]")); + assert!(rendered.body.omitted >= body.omitted); + } + } + } + + #[test] + fn bounded_pointer_prefix_claims_only_complete_labels_and_retries_the_rest() { + let mut instructions = Instructions::new(256, 256); + for i in 0..20 { + let path = + std::path::PathBuf::from(format!("/repo/rules/{i:04}-{}.md", "é".repeat(20))); + instructions.omitted.push(InstructionFile { + canonical: path.clone(), + path, + truncated: false, + }); + } + let mut seen = BTreeSet::new(); + for _ in 0..21 { + let Some(delivery) = instructions.reminder_delivery(&seen) else { + break; + }; + assert!(delivery.text.len() <= 256); + assert!(delivery.text.starts_with(OPEN) && delivery.text.ends_with(CLOSE)); + assert!(delivery.files.len() < 20); + for file in delivery.files { + assert!( + delivery + .text + .contains(&format!("\n- {}", file.path.display())) + ); + assert!(seen.insert(file.canonical.display().to_string())); + } + } + assert_eq!(seen.len(), 20); + } +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 5a85c3431..d080d6f85 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now use configured file/total text budgets, including lower nested limits. Complete rendered assembly accounting, trusted path-rule discovery/precedence and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Complete trusted path-rule discovery/precedence and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 937222bab..0fb4d161d 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1192,14 +1192,14 @@ Keep it short and specific to Cyber Code, and leave shared rules in `AGENTS.md`: - **Project files stay in the project.** A project instruction file, or an import from one, that resolves (after symlinks) outside the project root is not loaded and produces a warning, so a checkout cannot pull `~/.ssh/...` or `../other-repo` into the context, whether through `@path` or through a symlinked `CYBER.md`. A symlink inside the project (`CYBER.md -> AGENTS.md`) still works. Global files are yours and may import anywhere. Unresolved imports are warnings, not errors. - **Project files follow your `read` rules.** A project instruction file or import loads only when your `read` permission rules allow it outright. One they ask about or deny is skipped with a warning and no prompt, so under the default rules `@.env` (or a `CYBER.md` symlinked to `.env`) never reaches the context, and neither does a file a `read` deny covers. If the Location's config cannot be resolved, instruction observation is unavailable until it is fixed; the first Turn remains retryable without inference, and an existing Epoch retains its last available instruction snapshot. - **Only regular files.** A FIFO, device or directory in place of an instruction file is never opened for reading. In the baseline it makes the instructions unavailable, so the first Turn is blocked as for any unreadable instruction file (`cyber doctor` names it); found by a nested read, it is skipped with a warning. -- **Size.** `instruction_limits` accepts `max_file_bytes` (default 32768) and `max_total_bytes` (default 131072). Both must be positive integers, with the file limit no larger than the total. Each file's text is cut at the file limit with a `[Truncated: ...]` notice. Baseline text, imports included, consumes the total budget; later files are named as omitted. Labels and diagnostic notices are additional text. `cyber doctor` uses the same settings and reports truncation. Project definitions are withheld by the existing workspace-trust gate until approved, including definitions inside profiles. The object follows normal configuration layering by member, including profiles and command-line overrides; it does not change the existing concatenated `instructions` array or introduce a new managed-policy key. +- **Size.** `instruction_limits` accepts `max_file_bytes` (default 32768) and `max_total_bytes` (default 131072). Both must be positive integers, with the file limit no larger than the total. Each file's text is cut at the file limit with a `[Truncated: ...]` notice. The complete rendered instruction-file baseline (`core/instructions`), including file labels, imports, separators and truncation notices, consumes the total budget. Other Context Sources and the provider/agent system prefix have their own accounting. Files that cannot fit a complete label/notice are omitted with a doctor warning; a very small budget can withhold all file text. `cyber doctor` uses the same settings and reports truncation. Project definitions are withheld by the existing workspace-trust gate until approved, including definitions inside profiles. The object follows normal configuration layering by member, including profiles and command-line overrides; it does not change the existing concatenated `instructions` array or introduce a new managed-policy key. - **Edits mid-session** reach the model at the next Safe Boundary as one durable context update; the cached prompt prefix is not rewritten. ### Nested files on read Files in directories below the Location are not in the baseline. When the built-in `read` tool successfully reads a file under such a directory, that directory's instruction files (and those of the directories between) are appended to the result inside a `` block, once per file per Context Epoch; the attachment is recorded durably with the tool settlement, and a new epoch (for example after compaction) starts over. Reads through `bash`, `grep` or MCP tools do not trigger it, and neither do denied or failed reads. -One reminder carries at most the smaller of 32768 bytes and `instruction_limits.max_total_bytes` of instruction text. Files past that are named in the reminder (`read them with the read tool`) instead of attached. When even that reminder does not fit the tool output budget (`tool_output`), the read gets a short list of the file paths; when nothing fits, the read keeps its own output and the files can attach on a later read. Instruction files never turn a successful read into an error. Each attachment fires the `InstructionsLoaded` hook with `load_reason: "nested_read"` and the attached `file_paths`, as the baseline does with `session_start` and `changed`. +One complete reminder, including its wrapper, labels, notices and file list, fits the smaller of 32768 bytes and `instruction_limits.max_total_bytes`. Files beyond the content budget are named for an explicit read when their complete labels fit. Only files actually labelled in the delivered reminder count as attached; unlisted files remain eligible. Authorized import declarations are revisited under a bounded discovery budget so naming an importer does not strand its unlisted children. A cut final import line is ignored rather than interpreted as another filename. When even that reminder does not fit the tool output budget (`tool_output`), the read gets a short list of the file paths; when nothing fits, the read keeps its own output and the files can attach on a later read. Instruction files never turn a successful read into an error. Each attachment fires the `InstructionsLoaded` hook with `load_reason: "nested_read"` and the attached `file_paths`, as the baseline does with `session_start` and `changed`. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index db560c3af..837814706 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -22,7 +22,7 @@ Use the canonical 200 MiB per-Location fallback limit and seven-day default rete Load `.cyber/rules/*.md` and enabled `.claude/rules/*.md` at the existing directory precedence. Rules without `paths` join the baseline. Relative `paths` globs activate after a successful authorized read/edit/patch of a matching path; denied/failed operations do not consume once-per-Epoch delivery. Successful partial file settlements retain correct reminder ownership. Bind project root, definition digest, trust/config generation, successful operation identity and Context Epoch. Keep existing directory instruction ordering and `AGENTS.override.md` replacement semantics; do not use a project rule to widen tool/permission authority. -Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. Rendered labels, notices and omitted-file inventories still need full assembly accounting under task 4.1. +Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. ### 4. Root structured output is per admitted prompt diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index 5c439623b..fd0bd03ba 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -35,3 +35,9 @@ #### Scenario: Many imports - **WHEN** `/repo/CYBER.md` imports enough files to exceed the total instruction budget - **THEN** the baseline stops at the budget with a truncation notice, and a warning names each file left out + +#### Scenario: Complete instruction assembly is bounded +- **WHEN** instruction files, imports, labels or omitted-file lists exceed the configured total +- **THEN** the complete rendered instruction-file baseline (`core/instructions`) and each nested reminder SHALL fit their respective byte budgets, including labels, notices, separators and wrappers +- **AND** only complete file labels actually delivered SHALL consume reminder identity; unlisted imports SHALL remain discoverable after their importer is labelled +- **AND** a truncated import declaration SHALL NOT load the file named by its partial path diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 3e7414a24..888da3dd7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1576,3 +1576,48 @@ trusted path-rule discovery/globs, successful-operation/Epoch/digest-bound reminders, changed/revoked definitions, effective context inspection and the TUI flow remain required. Task 4.1 and all other parent tasks stay open; this does not accept rules/context, the recovery branch or issue #93. + +Rendered instruction assembly continuation (2026-11-10): `core/instructions` +now budgets complete file labels, bodies, separators and truncation notices. +Nested limits also include the system-reminder wrapper and bounded lists of whole +file labels. Tiny budgets withhold files instead of emitting partial labels or +notices. Typed bounded bodies retain their actual limiting budget and omitted +byte count, so UTF-8 cuts and per-file/aggregate notices remain truthful. Other +Context Sources and the provider/agent prefix have separate accounting. + +Reminder candidates carry the exact typed files they label. Durable settlement +IDs and InstructionsLoaded paths come from the successfully finalized candidate, +not the entire planned inventory or untrusted rendered text. In-group deduplication +occurs after finalization, so a failed attachment cannot reserve another call's +files. Bounded authorized rediscovery of labelled importers keeps unlisted children +eligible; a cut final import line is never interpreted as a different filename. +Read gates, confinement, regular-file checks, cycle/depth checks and bounded input +reads remain on rediscovery. Source-graph exhaustion/deep graph/restart acceptance +still belongs to the remaining discovery and provenance work. + +Verification passes: 449 server library tests (one explicit subprocess-fixture +entry ignored), 20 context/compaction tests, 10 real-App instruction tests and the +doctor regression. The App tests inspect actual provider text against the stored +source and enforce complete reminder byte limits. The new bounded-list App case +compares durable Epoch identities with complete labels in two actual tool results, +verifying later progress without consuming unlisted imports or repeating prior +identities. Unit coverage checks UTF-8/notice boundaries across small budgets, +bounded pointer prefixes, importer retry and truncated-path refusal. An earlier +assertion expecting the full 32 KiB body plus metadata was replaced by complete +rendered-size checks and positive content/identity checks. + +Affected server/App/CLI all-target Clippy with warnings denied, formatting, diff, +SDK generation consistency and workflow YAML/selection checks pass. Strict +OpenSpec 1.13.2 validates 100/100 items; lint reports 0 errors/25 warnings. Native C +deprecation and linker unwind warnings were observed without assigning origin. +Main was fetched with no missing upstream commits. The native workflow includes +assembly/reminder units and the new support-module filter, alongside real App +acceptance and retained logs. SDK runtime/full workspace and fresh-source platform +acceptance have not run for this increment. No API/event envelope changed. + +Focused code-review-skill review checks complete-boundary rendering, typed +delivery authority, post-finalization consumption, permission/confine checks on +rediscovery and partial-line refusal. Full rules/context review remains open for +trusted definition identity, source-graph exhaustion, operation/Epoch/restart and +revocation binding, effective inspection and the TUI flow. Task 4.1 and parent +acceptance remain incomplete; no goal or delivery completion is claimed. From c9e0c3736b862af9e4904d2719f5d002a84ef429 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 15:53:13 -0400 Subject: [PATCH 76/87] Load reviewed path rules with fenced definition identity --- .github/workflows/non-git-snapshot-proof.yml | 6 + crates/cyber-app/src/host.rs | 7 + crates/cyber-app/tests/instructions.rs | 195 +++++++ crates/cyber-cli/src/commands/doctor.rs | 2 + crates/cyber-core/src/config/load.rs | 8 +- crates/cyber-core/src/config/mod.rs | 3 + crates/cyber-core/src/lib.rs | 1 + crates/cyber-core/src/rules.rs | 373 +++++++++++++ crates/cyber-core/tests/config.rs | 31 ++ crates/cyber-server/src/runtime/context.rs | 1 + crates/cyber-server/src/runtime/drain.rs | 35 +- crates/cyber-server/src/runtime/events.rs | 3 + crates/cyber-server/src/runtime/host.rs | 5 + .../cyber-server/src/runtime/instructions.rs | 502 ++++++++++++++++-- .../src/runtime/instructions/assembly.rs | 3 +- crates/cyber-server/src/runtime/mod.rs | 1 + crates/cyber-server/src/runtime/model.rs | 11 + crates/cyber-tools/src/host.rs | 33 ++ crates/cyber-tools/tests/path_skills.rs | 1 + docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 22 + .../complete-daily-coding-surfaces/design.md | 4 + .../specs/system-context/spec.md | 13 + .../complete-daily-coding-surfaces/tasks.md | 57 ++ 24 files changed, 1274 insertions(+), 45 deletions(-) create mode 100644 crates/cyber-core/src/rules.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index f7403d08a..ee7febbb7 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -27,9 +27,12 @@ on: - crates/cyber-app/tests/snapshot_reclamation.rs - crates/cyber-app/tests/instructions.rs - crates/cyber-core/src/config/** + - crates/cyber-core/src/rules.rs + - crates/cyber-core/src/lib.rs - crates/cyber-core/tests/config.rs - crates/cyber-tools/src/host.rs - crates/cyber-tools/tests/instruction_config.rs + - crates/cyber-tools/tests/path_skills.rs - crates/cyber-server/src/runtime/context.rs - crates/cyber-server/src/runtime/instructions.rs - crates/cyber-server/src/runtime/instructions/** @@ -194,12 +197,15 @@ jobs: set -euo pipefail mkdir -p "$TMPDIR" cargo test --locked -p cyber-core --test config instruction_limits -- --nocapture 2>&1 | tee instruction-limits-proof.log + cargo test --locked -p cyber-core --lib rules:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-core --test config rule_definition_bytes -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::context:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::instructions:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::drain::skill_reminder_tests:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --test context_compaction -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-app --features native-git-acceptance --test instructions -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-tools --test instruction_config -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-tools --test path_skills -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::doctor::instruction_tests:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log - name: Retain native evidence if: always() diff --git a/crates/cyber-app/src/host.rs b/crates/cyber-app/src/host.rs index ab7b18ff5..611ca20d3 100644 --- a/crates/cyber-app/src/host.rs +++ b/crates/cyber-app/src/host.rs @@ -106,6 +106,13 @@ impl ToolHost for AppHost { self.builtin.instruction_read_gate(directory) } + fn instruction_rule_gate( + &self, + directory: &str, + ) -> Option { + self.builtin.instruction_rule_gate(directory) + } + fn finalize_skill_output( &self, directory: &str, diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index da86cd80e..c3ad4fc62 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -34,6 +34,7 @@ fn reply(body: &Value) -> Value { ] { if text.contains(prompt) { let arguments = json!({"path": "web/src/app.ts"}).to_string(); + let id = format!("{id}_{}", body["messages"].as_array().unwrap().len()); return json!({"tool_calls":[{"index":0,"id":id,"type":"function", "function":{"name":"read","arguments":arguments}}]}); } @@ -136,6 +137,183 @@ async fn bounded_lists_record_only_labelled_files_and_retry_unlisted_imports() { f.app.runtime.shutdown().await; } +#[tokio::test] +async fn project_rules_require_review_then_separate_baseline_from_matching_reads() { + let f = Fixture::start().await; + f.write(".cyber/rules/base.md", "ROOT_RULE_GUIDANCE"); + f.write( + ".cyber/rules/web.md", + "---\npaths: ['web/**']\n---\nWEB_RULE_GUIDANCE", + ); + f.write("web/src/app.ts", "export const app = true;"); + let held = f.session(&f.repo).await; + f.prompt(&held, &f.repo, "read the app first", 2).await; + assert!(!every_request(&f).contains("ROOT_RULE_GUIDANCE")); + assert!(!every_request(&f).contains("WEB_RULE_GUIDANCE")); + f.approve(&f.repo); + let session = f.session(&f.repo).await; + f.prompt(&session, &f.repo, "read the app first", 2).await; + let request = f.last_request(); + assert!(system_prompt(&request).contains("ROOT_RULE_GUIDANCE")); + assert!(!system_prompt(&request).contains("WEB_RULE_GUIDANCE")); + let result = last_tool_result(&request); + assert!(result.contains("WEB_RULE_GUIDANCE")); + assert!(!result.contains("paths:")); + let state = f.app.runtime.state(&session).await.unwrap(); + let ids = state.epoch.unwrap().reminded_rules; + assert_eq!(ids.len(), 1, "{ids:?}"); + assert!(ids.iter().all(|id| id.starts_with("rule:sha256:"))); + let events = f.app.store.read_events(&session, -1, 500).unwrap().events; + let settled = events + .iter() + .find(|event| { + event.kind == "session.tool.settled.1" + && event.data["rule_reminders"] + .as_array() + .is_some_and(|rules| !rules.is_empty()) + }) + .unwrap(); + assert!( + settled.data.get("instruction_reminders").is_none(), + "legacy path field is preserved" + ); + let binding = &settled.data["rule_reminders"][0]; + assert_eq!(binding["root"], f.repo.to_str().unwrap()); + assert_eq!( + binding["path"], + f.repo.join(".cyber/rules/web.md").to_str().unwrap() + ); + assert!(binding["digest"].as_str().unwrap().starts_with("sha256:")); + assert!( + binding["generation"] + .as_str() + .unwrap() + .starts_with("sha256:") + ); + let replayed = cyber_server::runtime::SessionState::replay(&events).unwrap(); + assert_eq!(replayed.epoch.unwrap().reminded_rules, ids); + f.prompt(&session, &f.repo, "read the app again", 2).await; + assert!(!last_tool_result(&f.last_request()).contains("WEB_RULE_GUIDANCE")); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn changed_rule_bytes_stay_held_until_the_new_definition_is_reviewed() { + let f = Fixture::start().await; + let path = ".cyber/rules/web.md"; + f.write(path, "---\npaths: ['web/**']\n---\nRULE_VERSION_A"); + f.write("web/src/app.ts", "export const app = true;"); + f.approve(&f.repo); + let session = f.session(&f.repo).await; + f.prompt(&session, &f.repo, "read the app first", 2).await; + assert!(last_tool_result(&f.last_request()).contains("RULE_VERSION_A")); + f.write(path, "---\npaths: ['web/**']\n---\nRULE_VERSION_B"); + f.prompt(&session, &f.repo, "read the app again", 2).await; + assert!(!last_tool_result(&f.last_request()).contains("RULE_VERSION_B")); + f.approve(&f.repo); + f.prompt(&session, &f.repo, "read the app again", 2).await; + assert!(last_tool_result(&f.last_request()).contains("RULE_VERSION_B")); + assert_eq!( + f.app + .runtime + .state(&session) + .await + .unwrap() + .epoch + .unwrap() + .reminded_rules + .len(), + 2 + ); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn rule_imports_keep_read_and_scope_gates_and_claude_compatibility() { + let f = Fixture::start_with( + json!({"compat":{"claude":false},"permissions":{"read":{".cyber/rules/secret.md":"deny"}}}), + ) + .await; + f.write( + ".cyber/rules/a.md", + "CYBER_GUIDANCE\n@../../docs/extra.md\n@../../../private.md\n@db.md", + ); + f.write( + ".cyber/rules/db.md", + "---\npaths: ['db/**']\n---\nDB_GUIDANCE", + ); + f.write(".cyber/rules/secret.md", "SECRET_GUIDANCE"); + f.write(".claude/rules/a.md", "CLAUDE_GUIDANCE"); + f.write("docs/extra.md", "EXTRA_GUIDANCE"); + std::fs::write( + f.repo.parent().unwrap().join("private.md"), + "OUTSIDE_GUIDANCE", + ) + .unwrap(); + std::fs::create_dir_all(f.config.join("rules")).unwrap(); + std::fs::write(f.config.join("rules/global.md"), "GLOBAL_GUIDANCE").unwrap(); + f.approve(&f.repo); + let session = f.session(&f.repo).await; + f.prompt(&session, &f.repo, "hello", 1).await; + let system = system_prompt(&f.last_request()); + for present in ["GLOBAL_GUIDANCE", "CYBER_GUIDANCE", "EXTRA_GUIDANCE"] { + assert!(system.contains(present), "{present}: {system}"); + } + for absent in [ + "DB_GUIDANCE", + "SECRET_GUIDANCE", + "CLAUDE_GUIDANCE", + "OUTSIDE_GUIDANCE", + ] { + assert!(!system.contains(absent), "{absent}: {system}"); + } + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn nested_rules_use_the_reviewed_operation_directory_in_the_same_checkout() { + let f = Fixture::start().await; + f.write( + "web/.cyber/rules/local.md", + "---\npaths: ['web/src/**']\n---\nLOCAL_NESTED_GUIDANCE", + ); + f.write("web/src/app.ts", "export const app = true;"); + f.approve(&f.repo.join("web")); + let session = f.session(&f.repo).await; + f.prompt(&session, &f.repo, "read the app first", 2).await; + assert!(!system_prompt(&f.last_request()).contains("LOCAL_NESTED_GUIDANCE")); + assert!(last_tool_result(&f.last_request()).contains("LOCAL_NESTED_GUIDANCE")); + f.app.runtime.shutdown().await; +} + +#[tokio::test] +async fn a_nested_checkout_cannot_supply_rules_for_its_parent_session() { + let f = Fixture::start().await; + f.write("web/.git/marker", ""); + f.write( + "web/.cyber/rules/local.md", + "---\npaths: ['web/src/**']\n---\nFOREIGN_RULE_GUIDANCE", + ); + f.write("web/src/app.ts", "export const app = true;"); + f.approve(&f.repo.join("web")); + let session = f.session(&f.repo).await; + f.prompt(&session, &f.repo, "read the app first", 2).await; + assert!(last_tool_result(&f.last_request()).contains("export const app")); + assert!(!every_request(&f).contains("FOREIGN_RULE_GUIDANCE")); + assert!( + f.app + .runtime + .state(&session) + .await + .unwrap() + .epoch + .unwrap() + .reminded_rules + .is_empty() + ); + f.app.runtime.shutdown().await; +} + struct Fixture { _temp: tempfile::TempDir, repo: PathBuf, @@ -147,6 +325,23 @@ struct Fixture { } impl Fixture { + fn approve(&self, directory: &Path) { + let env = std::collections::HashMap::::new(); + let home = self._temp.path().join("home"); + let report = cyber_core::config::trust_report(&cyber_core::config::LoadRequest { + location: directory, + paths: &self.app.paths, + env: &env, + home: &home, + profile: None, + overrides: &[], + flags: json!({}), + }) + .unwrap(); + cyber_core::trust::TrustStore::new(self.app.paths.trust_file()) + .approve(&report.checkout_root, report.digest.as_deref().unwrap()) + .unwrap(); + } async fn start() -> Self { Self::start_with(json!({})).await } diff --git a/crates/cyber-cli/src/commands/doctor.rs b/crates/cyber-cli/src/commands/doctor.rs index 10aa8931b..0741066e4 100644 --- a/crates/cyber-cli/src/commands/doctor.rs +++ b/crates/cyber-cli/src/commands/doctor.rs @@ -149,6 +149,8 @@ fn instructions(ctx: &Context, config: Option<&cyber_core::config::Resolved>) -> ); } } + settings.rule_gate = config + .map(|config| cyber_server::runtime::instructions::RuleGate::from_report(&config.trust)); settings.read_gate = Some(cyber_tools::permissions::instruction_read_gate( config.map(|c| (&c.value, &c.sources)), &ctx.location, diff --git a/crates/cyber-core/src/config/load.rs b/crates/cyber-core/src/config/load.rs index 746a446d7..4a4b5cf7a 100644 --- a/crates/cyber-core/src/config/load.rs +++ b/crates/cyber-core/src/config/load.rs @@ -642,6 +642,7 @@ fn assess_trust(req: &LoadRequest<'_>, project: &[Layer]) -> Result, project: &[Layer]) -> Result, project: &[Layer]) -> Result, project: &[Layer]) -> Result, /// Canonical checkout root the approval is keyed by. pub checkout_root: PathBuf, /// `sha256:` of the security-sensitive definitions, absent when there are none. diff --git a/crates/cyber-core/src/lib.rs b/crates/cyber-core/src/lib.rs index 28ec58de1..f0066a883 100644 --- a/crates/cyber-core/src/lib.rs +++ b/crates/cyber-core/src/lib.rs @@ -28,6 +28,7 @@ pub mod project; pub mod prompts; pub mod redact; pub mod review; +pub mod rules; pub mod skills; pub mod teams; pub mod trust; diff --git a/crates/cyber-core/src/rules.rs b/crates/cyber-core/src/rules.rs new file mode 100644 index 000000000..8f3db4b5d --- /dev/null +++ b/crates/cyber-core/src/rules.rs @@ -0,0 +1,373 @@ +//! Fenced rule definitions and the exact bytes covered by workspace approval. +use crate::import::{SourceSnapshot, command_directory}; +use globset::{GlobBuilder, GlobMatcher}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +pub const MAX_FILES: usize = 128; +pub const MAX_TOTAL_BYTES: usize = 4 * 1024 * 1024; +const MAX_ENTRIES: usize = 4096; +const MAX_HEADER: usize = 16 * 1024; + +pub struct Definition { + pub path: PathBuf, + pub digest: String, + pub body: String, + pub paths: Option>, + matchers: Vec, + snapshot: SourceSnapshot, +} + +impl Definition { + pub fn read(path: &Path) -> Result { + let snapshot = SourceSnapshot::read_command(path).map_err(|e| e.reason.to_owned())?; + let (frontmatter, body) = parse(snapshot.text().map_err(|e| e.reason.to_owned())?)?; + let matchers = frontmatter.paths.as_ref().map_or(Ok(Vec::new()), |paths| { + paths + .iter() + .map(|pattern| glob(pattern)) + .collect::, _>>() + })?; + let definition = Self { + path: path.into(), + digest: digest(snapshot.bytes()), + body: body.to_owned(), + paths: frontmatter.paths, + matchers, + snapshot, + }; + definition.verify()?; + Ok(definition) + } + + pub fn verify(&self) -> Result<(), String> { + self.snapshot.verify().map_err(|e| e.reason.to_owned()) + } + + pub fn byte_len(&self) -> usize { + self.snapshot.bytes().len() + } + + pub fn applies(&self, root: &Path, target: Option<&Path>) -> bool { + if self.paths.is_none() { + return true; + } + let Some(relative) = target + .and_then(|target| target.strip_prefix(root).ok()) + .and_then(Path::to_str) + else { + return false; + }; + let relative = relative.replace('\\', "/"); + self.matchers + .iter() + .any(|matcher| matcher.is_match(&relative)) + } +} + +fn digest(bytes: &[u8]) -> String { + format!("sha256:{:x}", Sha256::digest(bytes)) +} + +#[derive(Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct Frontmatter { + #[serde(default, deserialize_with = "paths")] + paths: Option>, + #[serde(default)] + description: Option, +} + +fn paths<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result>, D::Error> { + Vec::::deserialize(deserializer).map(Some) +} + +fn parse(text: &str) -> Result<(Frontmatter, &str), String> { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let Some(rest) = text + .strip_prefix("---\n") + .or_else(|| text.strip_prefix("---\r\n")) + else { + return Ok((Frontmatter::default(), text)); + }; + let mut offset = 0; + for line in rest.split_inclusive('\n') { + if offset > MAX_HEADER { + return Err("rule frontmatter exceeds 16 KiB".into()); + } + if line.trim_end_matches(['\r', '\n']) == "---" { + let header = &rest[..offset]; + let fm = if header.trim().is_empty() { + Frontmatter::default() + } else { + let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(header) + .map_err(|e| format!("invalid rule frontmatter: {e}"))?; + if !value.is_mapping() { + return Err("rule frontmatter must be a mapping".into()); + } + serde_yaml_ng::from_value::(value) + .map_err(|e| format!("invalid rule frontmatter: {e}"))? + }; + if fm + .description + .as_ref() + .is_some_and(|text| text.len() > 1024) + { + return Err("rule description exceeds 1024 bytes".into()); + } + if fm + .paths + .as_ref() + .is_some_and(|paths| paths.is_empty() || paths.len() > 64) + { + return Err("rule paths must contain 1-64 project-relative globs".into()); + } + return Ok((fm, &rest[offset + line.len()..])); + } + offset += line.len(); + } + Err("rule frontmatter has no closing delimiter within 16 KiB".into()) +} + +fn glob(pattern: &str) -> Result { + if pattern.len() > 256 { + return Err("rule paths must be bounded project-relative globs".into()); + } + let pattern = pattern.strip_prefix("./").unwrap_or(pattern); + if pattern.is_empty() + || pattern.len() > 256 + || pattern.starts_with('/') + || pattern.contains(['\\', ':']) + || pattern.chars().any(char::is_control) + || pattern.split('/').any(|part| part == ".." || part == ".") + { + return Err("rule paths must be bounded project-relative globs".into()); + } + GlobBuilder::new(pattern) + .literal_separator(true) + .build() + .map(|glob| glob.compile_matcher()) + .map_err(|e| format!("invalid rule glob: {e}")) +} + +/// Direct Markdown children only; no linked directory chain or unbounded enumeration. +pub fn candidates(directory: &Path) -> Result, String> { + match std::fs::symlink_metadata(directory) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e.to_string()), + Ok(_) => (), + } + let chain = command_directory(directory).map_err(|e| e.reason.to_owned())?; + let entries = chain.last().unwrap().entries().map_err(|e| e.to_string())?; + let mut paths = Vec::new(); + for (index, entry) in entries.enumerate() { + if index >= MAX_ENTRIES { + return Err("rule directory entry bound exceeded".into()); + } + let entry = entry.map_err(|e| e.to_string())?; + let path = directory.join(entry.file_name()); + if path.extension().is_some_and(|ext| ext == "md") { + if path.to_str().is_none() { + return Err("rule paths must be UTF-8".into()); + } + paths.push(path); + } + if paths.len() > MAX_FILES { + return Err("rule file bound exceeded".into()); + } + } + paths.sort(); + Ok(paths) +} + +pub fn project_directories(location: &Path) -> Vec { + let location = location.canonicalize().unwrap_or_else(|_| location.into()); + let root = crate::config::project_root(&location); + let mut dirs: Vec<_> = location + .ancestors() + .take_while(|dir| dir.starts_with(&root)) + .map(Path::to_path_buf) + .collect(); + dirs.reverse(); + dirs.into_iter() + .flat_map(|dir| [dir.join(".claude/rules"), dir.join(".cyber/rules")]) + .collect() +} + +pub struct TrustInventory { + pub definitions: Vec<(String, PathBuf, String)>, + pub digests: BTreeMap, +} + +/// Invalid/unknown inventories have no authority. Their marker still invalidates an +/// earlier approval; later repair requires review of the now-readable bytes. +pub fn trust_subset(location: &Path) -> TrustInventory { + let root = crate::config::project_root(location); + let root = root.canonicalize().unwrap_or(root); + let mut subset = Vec::new(); + let mut digests = BTreeMap::new(); + let mut total = 0usize; + let mut count = 0; + for directory in project_directories(location) { + let files = match candidates(&directory) { + Ok(files) => files, + Err(reason) => { + subset.push(( + directory.display().to_string(), + directory, + format!("refused:{reason}"), + )); + continue; + } + }; + for path in files { + count += 1; + if count > MAX_FILES { + return TrustInventory { + definitions: vec![( + "rules:unavailable".into(), + root, + "rule inventory bound exceeded".into(), + )], + digests: BTreeMap::new(), + }; + } + let value = match SourceSnapshot::read_command(&path) { + Ok(snapshot) => { + total = total.saturating_add(snapshot.bytes().len()); + if count > MAX_FILES || total > MAX_TOTAL_BYTES { + return TrustInventory { + definitions: vec![( + "rules:unavailable".into(), + root, + "rule inventory bound exceeded".into(), + )], + digests: BTreeMap::new(), + }; + } + let value = digest(snapshot.bytes()); + digests.insert(path.clone(), value.clone()); + value + } + Err(error) => format!("refused:{}", error.reason), + }; + subset.push(( + path.strip_prefix(&root) + .unwrap_or(&path) + .display() + .to_string(), + path, + value, + )); + } + } + TrustInventory { + definitions: subset, + digests, + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn root() -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::create_dir_all(root.join(".cyber/rules")).unwrap(); + (temp, root) + } + #[test] + fn scopes_are_relative_to_the_project_and_unscoped_rules_are_baseline() { + let (_temp, root) = root(); + let path = root.join(".cyber/rules/migrations.md"); + std::fs::write( + &path, + "---\npaths: [\"./db/migrations/**\"]\n---\nUse transactions.", + ) + .unwrap(); + let rule = Definition::read(&path).unwrap(); + assert_eq!(rule.body, "Use transactions."); + assert!(!rule.applies(&root, None)); + assert!(rule.applies(&root, Some(&root.join("db/migrations/0042.sql")))); + assert!(!rule.applies(&root, Some(&root.join("db/other.sql")))); + assert!(!rule.applies( + &root, + Some(&root.parent().unwrap().join("db/migrations/0042.sql")) + )); + std::fs::write(&path, "plain Markdown").unwrap(); + assert!(Definition::read(&path).unwrap().applies(&root, None)); + assert!( + rule.verify().is_err(), + "captured definition cannot silently change" + ); + } + #[test] + fn invalid_frontmatter_never_falls_back_to_unscoped_guidance() { + let (_temp, root) = root(); + let path = root.join(".cyber/rules/a.md"); + for header in [ + "[]", + "paths: null", + "paths: []", + "paths: [../secrets/**]", + "paths: ['/absolute/**']", + "paths: ['C:/private/**']", + "paths: ['[']", + "paths: ['a\\b']", + "permissions: allow", + "paths: [3]", + ] { + std::fs::write(&path, format!("---\n{header}\n---\nNever baseline")).unwrap(); + assert!(Definition::read(&path).is_err(), "{header}"); + } + std::fs::write( + &path, + format!( + "---\ndescription: {}\n---\nbody", + "a".repeat(MAX_HEADER + 1) + ), + ) + .unwrap(); + assert!(Definition::read(&path).is_err()); + } + #[test] + fn trust_inventory_tracks_full_definition_bytes_and_bounds_the_directory() { + let (_temp, root) = root(); + let path = root.join(".cyber/rules/a.md"); + std::fs::write(&path, "---\npaths: ['a/**']\n---\nbody").unwrap(); + let first = trust_subset(&root); + assert_eq!( + first.digests[&path], + Definition::read(&path).unwrap().digest + ); + std::fs::write(&path, "---\npaths: ['b/**']\n---\nbody").unwrap(); + assert_ne!(first.definitions, trust_subset(&root).definitions); + for i in 0..MAX_FILES { + std::fs::write(root.join(format!(".cyber/rules/{i}.md")), "rule").unwrap(); + } + assert!(candidates(&root.join(".cyber/rules")).is_err()); + assert!(trust_subset(&root).digests.is_empty()); + } + #[cfg(unix)] + #[test] + fn linked_definitions_and_directories_have_no_rule_authority() { + use std::os::unix::fs::symlink; + let (_temp, root) = root(); + let source = root.join("source.md"); + std::fs::write(&source, "private").unwrap(); + let linked = root.join(".cyber/rules/link.md"); + symlink(&source, &linked).unwrap(); + assert!(Definition::read(&linked).is_err()); + assert!(!trust_subset(&root).digests.contains_key(&linked)); + std::fs::remove_file(&linked).unwrap(); + std::fs::hard_link(&source, &linked).unwrap(); + assert!(Definition::read(&linked).is_err()); + symlink(root.join(".cyber/rules"), root.join("alias")).unwrap(); + assert!(candidates(&root.join("alias")).is_err()); + } +} diff --git a/crates/cyber-core/tests/config.rs b/crates/cyber-core/tests/config.rs index 6154700e8..5dec22bc6 100644 --- a/crates/cyber-core/tests/config.rs +++ b/crates/cyber-core/tests/config.rs @@ -490,6 +490,37 @@ fn instructions_concatenate_global_first() { ); } +#[test] +fn rule_definition_bytes_require_checkout_approval_and_edits_revoke_it() { + let mut f = Fixture::new(); + let rule = f.write( + ".cyber/rules/scoped.md", + "---\npaths: ['web/**']\n---\nUse the web convention.", + ); + let rule = rule.canonicalize().unwrap(); + let held = f.load().unwrap(); + assert!(!held.trust.trusted); + assert!(held.trust.definitions.contains(&rule.display().to_string())); + assert!(held.trust.rule_digests.contains_key(&rule)); + f.approve_current(); + let approved = f.load().unwrap(); + assert!(approved.trust.trusted); + f.write( + ".cyber/rules/scoped.md", + "---\npaths: ['db/**']\n---\nUse the web convention.", + ); + let changed = f.load().unwrap(); + assert!(!changed.trust.trusted); + assert_ne!( + approved.trust.rule_digests[&rule], + changed.trust.rule_digests[&rule] + ); + f.env + .insert("CYBER_DISABLE_PROJECT_CONFIG".into(), "1".into()); + let disabled = f.load().unwrap(); + assert!(disabled.trust.trusted && disabled.trust.rule_digests.is_empty()); +} + #[test] fn instruction_limits_layer_by_member_without_changing_instruction_arrays() { use config::InstructionLimits; diff --git a/crates/cyber-server/src/runtime/context.rs b/crates/cyber-server/src/runtime/context.rs index 1685bfe28..c55a5123e 100644 --- a/crates/cyber-server/src/runtime/context.rs +++ b/crates/cyber-server/src/runtime/context.rs @@ -261,6 +261,7 @@ mod tests { max_total_bytes: super::super::instructions::DEFAULT_MAX_TOTAL_BYTES, home: None, read_gate: None, + rule_gate: None, }), today: Some("2026-10-03".into()), } diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 914401819..e37b61a9e 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -955,6 +955,7 @@ impl Inner { &cyber_core::env::ProcessEnv, ); settings.read_gate = self.tools.instruction_read_gate(directory); + settings.rule_gate = self.tools.instruction_rule_gate(directory); settings.set_limits(limits); Ok(settings) } @@ -2313,6 +2314,7 @@ fn settled( skill_activation: None, skill_reminders: Vec::new(), instruction_reminders: Vec::new(), + rule_reminders: Vec::new(), structured_output: None, call_id: call_id.into(), status, @@ -2371,7 +2373,12 @@ impl Reminded { .as_ref() .map_or_else(Self::default, |epoch| Self { skills: epoch.reminded_skills.clone(), - instructions: epoch.reminded_instructions.clone(), + instructions: epoch + .reminded_instructions + .iter() + .chain(&epoch.reminded_rules) + .cloned() + .collect(), }) } } @@ -2422,9 +2429,22 @@ pub(super) fn settlement_with_reminders( delivery .files .iter() - .map(|file| file.canonical.display().to_string()) + .filter(|file| file.rule_binding.is_none()) + .map(|file| file.identity()) .collect::>() }); + let rule_bindings = instructions.as_ref().map_or_else(Vec::new, |delivery| { + delivery + .files + .iter() + .filter_map(|file| file.rule_binding.clone()) + .collect() + }); + seen.instructions.extend( + rule_bindings + .iter() + .map(super::instructions::RuleBinding::identity), + ); seen.skills.extend(names.iter().cloned()); seen.instructions.extend(attached.iter().cloned()); let loaded = instructions.map(|delivery| super::context::LoadedInstructions { @@ -2442,6 +2462,7 @@ pub(super) fn settlement_with_reminders( skill_activation: None, skill_reminders: names, instruction_reminders: attached, + rule_reminders: rule_bindings, structured_output: value, call_id: call_id.into(), status: if failed { @@ -2540,6 +2561,7 @@ pub(super) fn settlement( skill_activation: Some(activation), skill_reminders: Vec::new(), instruction_reminders: Vec::new(), + rule_reminders: Vec::new(), structured_output: None, call_id: call_id.into(), status: CallStatus::Ok, @@ -2572,6 +2594,7 @@ pub(super) fn settlement( skill_activation: None, skill_reminders: Vec::new(), instruction_reminders: Vec::new(), + rule_reminders: Vec::new(), structured_output: Some(value), call_id: call_id.into(), status: CallStatus::Ok, @@ -2595,6 +2618,7 @@ pub(super) fn settlement( skill_activation: None, skill_reminders: Vec::new(), instruction_reminders: Vec::new(), + rule_reminders: Vec::new(), structured_output: value, call_id: call_id.into(), status: if failed { @@ -2712,6 +2736,13 @@ mod skill_reminder_tests { use super::super::host::{SkillSuggestion, ToolScope}; use super::*; + #[test] + fn legacy_settlements_keep_canonical_paths_and_have_no_fabricated_rule_bindings() { + let settled: ToolSettled = serde_json::from_value(serde_json::json!({"call_id":"old","status":"ok","output":"text","instruction_reminders":["/repo/web/CYBER.md"]})).unwrap(); + assert_eq!(settled.instruction_reminders, ["/repo/web/CYBER.md"]); + assert!(settled.rule_reminders.is_empty()); + } + #[test] fn settlement_preserves_typed_null_and_uses_metadata_instead_of_output_text() { let def = ToolDef { diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 3cc3ce1b9..33e0b1411 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -375,6 +375,9 @@ pub struct ToolSettled { /// Canonical paths of nested instruction files attached to this result. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub instruction_reminders: Vec, + /// Approved rule definitions actually labelled in this successful result. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub rule_reminders: Vec, #[serde( default, skip_serializing_if = "Option::is_none", diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index b39facf5e..dbcf6a66e 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -924,6 +924,11 @@ pub trait ToolHost: Send + Sync { None } + /// Exact approved project rule definitions; absent hosts hold project rules. + fn instruction_rule_gate(&self, _directory: &str) -> Option { + None + } + /// Finalize the complete reminder and tool text before recording delivered skill names. fn finalize_skill_output( &self, diff --git a/crates/cyber-server/src/runtime/instructions.rs b/crates/cyber-server/src/runtime/instructions.rs index 5e9e13d34..6ffd4793a 100644 --- a/crates/cyber-server/src/runtime/instructions.rs +++ b/crates/cyber-server/src/runtime/instructions.rs @@ -52,8 +52,81 @@ pub struct InstructionSettings { /// Which project files (and imports of them) may be read; `None` allows every one, /// for hosts without a permission policy. pub read_gate: Option, + pub rule_gate: Option, } +#[derive(Debug, Clone)] +pub struct RuleAuthorization { + pub root: PathBuf, + pub digest: String, + pub generation: String, +} +impl RuleAuthorization { + pub fn from_report(report: &cyber_core::config::TrustReport, path: &Path) -> Option { + if !report.trusted { + return None; + } + Some(Self { + root: report.checkout_root.clone(), + digest: report.rule_digests.get(path)?.clone(), + generation: report.digest.clone()?, + }) + } +} + +/// Definition identity retained separately from legacy canonical-path reminders. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RuleBinding { + pub root: String, + pub path: String, + pub digest: String, + pub generation: String, +} +impl RuleBinding { + pub fn identity(&self) -> String { + use sha2::{Digest, Sha256}; + let mut hash = Sha256::new(); + hash.update(b"cyber-rule-reminder-v1\0"); + for field in [&self.root, &self.path, &self.digest, &self.generation] { + hash.update((field.len() as u64).to_le_bytes()); + hash.update(field.as_bytes()); + } + format!("rule:sha256:{hash:x}", hash = hash.finalize()) + } +} + +type RuleCheck = dyn Fn(&Path, &Path) -> Result, String> + Send + Sync; +#[derive(Clone)] +pub struct RuleGate(Arc); +impl RuleGate { + pub fn new( + check: impl Fn(&Path, &Path) -> Result, String> + + Send + + Sync + + 'static, + ) -> Self { + Self(Arc::new(check)) + } + pub fn from_report(report: &cyber_core::config::TrustReport) -> Self { + let report = report.clone(); + Self::new(move |path, _scope| Ok(RuleAuthorization::from_report(&report, path))) + } + fn authorize(&self, path: &Path, scope: &Path) -> Result, String> { + (self.0)(path, scope) + } +} +impl std::fmt::Debug for RuleGate { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("RuleGate") + } +} +impl PartialEq for RuleGate { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} +impl Eq for RuleGate {} + /// Whether a project instruction file may enter the model context, given its path as /// discovered (or as an import resolved it) and its canonical path. Only `true` loads it, /// so a file the Location's `read` rules would ask about or deny (`.env` through an @@ -104,6 +177,7 @@ impl InstructionSettings { max_total_bytes: DEFAULT_MAX_TOTAL_BYTES, home: cyber_core::paths::home_dir(env), read_gate: None, + rule_gate: None, } } @@ -122,6 +196,15 @@ pub struct InstructionFile { /// The deduplication key. pub canonical: PathBuf, pub truncated: bool, + pub(crate) rule_binding: Option, +} +impl InstructionFile { + pub(crate) fn identity(&self) -> String { + self.rule_binding.as_ref().map_or_else( + || self.canonical.display().to_string(), + RuleBinding::identity, + ) + } } /// Bounded instruction bodies, the files behind them, and non-fatal problems. @@ -174,8 +257,7 @@ impl Instructions { } pub(crate) fn reminder_delivery(&self, seen: &BTreeSet) -> Option { - let eligible = - |file: &&InstructionFile| !seen.contains(&file.canonical.display().to_string()); + let eligible = |file: &&InstructionFile| !seen.contains(&file.identity()); // Reserve complete labels first when some files could not enter the body. // A bounded prefix is truthful: unlisted files remain eligible next time. let pointers = (!self.omitted.is_empty()) @@ -193,7 +275,7 @@ impl Instructions { let (mut text, mut files) = assembly::blocks( self.blocks .iter() - .filter(|block| !seen.contains(&block.file.canonical.display().to_string())), + .filter(|block| !seen.contains(&block.file.identity())), self.max_file_bytes, self.content_budget().saturating_sub(reserved), ); @@ -213,7 +295,7 @@ impl Instructions { self.files .iter() .chain(&self.omitted) - .filter(|file| !seen.contains(&file.canonical.display().to_string())), + .filter(|file| !seen.contains(&file.identity())), self.content_budget(), )?; assembly::finish(pointer.text, pointer.files) @@ -223,11 +305,7 @@ impl Instructions { pub fn attached(&self) -> Vec { self.reminder_delivery(&BTreeSet::new()) .map_or_else(Vec::new, |delivery| { - delivery - .files - .iter() - .map(|file| file.canonical.display().to_string()) - .collect() + delivery.files.iter().map(|file| file.identity()).collect() }) } } @@ -294,26 +372,38 @@ pub fn load_nested( let dirs = dir .map(|dir| project_dirs(&location, dir)) .unwrap_or_default(); - if !settings.project || dirs.is_empty() { + if dirs.is_empty() { return Instructions::default(); } let mut baseline = Loader::new(settings, settings.max_total_bytes); let _ = fill_baseline(&mut baseline, global_dir, &location); let budget = NESTED_MAX_TOTAL_BYTES.min(settings.max_total_bytes); let mut loader = Loader::new(settings, budget); + loader.rule_root = canonical(&cyber_core::config::project_root(&location)); + loader.rule_scope = dir.unwrap_or(location.as_path()).to_path_buf(); + loader.rule_target = Some(target.clone()); + loader.global_rules = canonical(global_dir).join("rules"); loader.delivered = baseline .out .files .iter() - .map(|file| file.canonical.clone()) + .map(InstructionFile::identity) .collect(); - loader.delivered.extend(attached.iter().map(PathBuf::from)); + loader.delivered.extend(attached.iter().cloned()); loader.remaining = budget.saturating_sub(assembly::OPEN.len() + assembly::CLOSE.len()); let root = canonical(&cyber_core::config::project_root(&location)); - for file in dirs.iter().flat_map(|dir| directory_files(dir, settings)) { - if let Err(error) = loader.load(&file, Some(&root)) { - loader.out.warnings.push(error); + let global_rules = loader.global_rules.clone(); + loader.rules(&global_rules, &root, Some(&target), false); + for dir in dirs { + if !settings.project { + break; } + for file in directory_files(&dir, settings) { + if let Err(error) = loader.load(&file, Some(&root)) { + loader.out.warnings.push(error); + } + } + loader.project_rules(&dir, &root, Some(&target)); } loader.out } @@ -341,23 +431,29 @@ pub fn read_target(location: &Path, home: Option<&Path>, path: &str) -> PathBuf /// Load every baseline file; returns the first read error, if any. fn fill_baseline(loader: &mut Loader<'_>, global_dir: &Path, location: &Path) -> Option { let mut first = None; + loader.rule_root = canonical(&cyber_core::config::project_root(location)); + loader.global_rules = canonical(global_dir).join("rules"); + loader.rule_scope = location.into(); for file in GLOBAL_FILES.iter().map(|name| global_dir.join(name)) { if let Err(error) = loader.load(&file, None) { first.get_or_insert(error); } } + let project_root = canonical(&cyber_core::config::project_root(location)); + let global_rules = loader.global_rules.clone(); + loader.rules(&global_rules, &project_root, None, false); if !loader.settings.project { return first; } let root = canonical(&cyber_core::config::project_root(location)); let dirs = project_dirs(location, location); - for file in dirs - .iter() - .flat_map(|dir| directory_files(dir, loader.settings)) - { - if let Err(error) = loader.load(&file, Some(&root)) { - first.get_or_insert(error); + for dir in dirs { + for file in directory_files(&dir, loader.settings) { + if let Err(error) = loader.load(&file, Some(&root)) { + first.get_or_insert(error); + } } + loader.project_rules(&dir, &root, None); } first } @@ -365,9 +461,16 @@ fn fill_baseline(loader: &mut Loader<'_>, global_dir: &Path, location: &Path) -> /// Reads each file once by canonical path, expanding `@path` imports depth first. struct Loader<'a> { settings: &'a InstructionSettings, - seen: BTreeSet, - delivered: BTreeSet, + seen: BTreeSet, + delivered: BTreeSet, discovery_remaining: usize, + rule_files: usize, + rule_seen: BTreeSet, + rule_bytes: usize, + rule_root: PathBuf, + rule_scope: PathBuf, + rule_target: Option, + global_rules: PathBuf, /// Canonical paths of the files whose imports are being expanded. stack: Vec, /// Bytes of instruction text still allowed. @@ -382,12 +485,171 @@ impl<'a> Loader<'a> { seen: BTreeSet::new(), delivered: BTreeSet::new(), discovery_remaining: settings.max_total_bytes, + rule_files: 0, + rule_seen: BTreeSet::new(), + rule_bytes: 0, + rule_root: PathBuf::new(), + rule_scope: PathBuf::new(), + rule_target: None, + global_rules: PathBuf::new(), stack: Vec::new(), remaining: budget, out: Instructions::new(settings.max_bytes, budget), } } + fn project_rules(&mut self, directory: &Path, root: &Path, target: Option<&Path>) { + if self.settings.claude_md { + self.rules(&directory.join(".claude/rules"), root, target, true); + } + self.rules(&directory.join(".cyber/rules"), root, target, true); + } + + fn rules(&mut self, directory: &Path, root: &Path, target: Option<&Path>, project: bool) { + let files = match cyber_core::rules::candidates(directory) { + Ok(files) => files, + Err(error) => { + self.out + .warnings + .push(format!("{}: {error}", directory.display())); + return; + } + }; + for path in files { + if self.rule_files >= 2 * cyber_core::rules::MAX_FILES + || self.rule_bytes >= cyber_core::rules::MAX_TOTAL_BYTES + { + self.out.warnings.push(format!( + "{}: rule discovery bound exceeded", + directory.display() + )); + break; + } + if let Err(error) = self.rule(&path, root, target, project) { + self.out + .warnings + .push(format!("{}: {error}", path.display())); + } + } + } + + fn rule( + &mut self, + path: &Path, + root: &Path, + target: Option<&Path>, + project: bool, + ) -> Result<(), String> { + if !self.rule_seen.insert(path.into()) { + return Ok(()); + } + self.rule_files += 1; + if self.rule_files > 2 * cyber_core::rules::MAX_FILES { + return Err("rule file bound exceeded".into()); + } + if project + && !self.settings.claude_md + && path + .parent() + .and_then(Path::parent) + .and_then(Path::file_name) + .is_some_and(|name| name.to_string_lossy().eq_ignore_ascii_case(".claude")) + { + return Err("Claude rule compatibility is disabled".into()); + } + // Approval and read policy precede opening project definition bytes. + let authorization = if project { + if !path.starts_with(root) || !self.settings.read_allowed(path, &canonical(path)) { + return Err("rule definition is not allowed by read policy".into()); + } + let authorization = self + .settings + .rule_gate + .as_ref() + .map(|gate| gate.authorize(path, &self.rule_scope)) + .transpose()? + .flatten() + .ok_or( + "project rule is held until its definition is trusted (cyber trust inspect)", + )?; + if authorization.root != root { + return Err("rule approval belongs to another project root".into()); + } + Some(authorization) + } else { + None + }; + let definition = cyber_core::rules::Definition::read(path)?; + self.rule_bytes = self.rule_bytes.saturating_add(definition.byte_len()); + if self.rule_bytes > cyber_core::rules::MAX_TOTAL_BYTES { + return Err("rule definition byte bound exceeded".into()); + } + if authorization + .as_ref() + .is_some_and(|auth| auth.digest != definition.digest) + { + return Err("rule definition differs from its approved digest".into()); + } + if !definition.applies(root, target) { + return Ok(()); + } + let generation = authorization + .as_ref() + .map_or("owner-global", |auth| auth.generation.as_str()); + let binding = RuleBinding { + root: root.to_str().ok_or("rule root must be UTF-8")?.into(), + path: path.to_str().ok_or("rule path must be UTF-8")?.into(), + digest: definition.digest.clone(), + generation: generation.into(), + }; + let identity = binding.identity(); + if !self.seen.insert(identity.clone()) { + return Ok(()); + } + if self.delivered.contains(&identity) { + let limit = self.settings.max_bytes.min(self.discovery_remaining); + let body = bounded_text(&definition.body, limit); + self.discovery_remaining -= body.text.len(); + definition.verify()?; + return self.expand_rule_imports(path, root, project, &body); + } + let file = InstructionFile { + path: path.into(), + canonical: path.into(), + truncated: false, + rule_binding: Some(binding), + }; + let label = format!("{INSTRUCTIONS_FROM}{}\n", path.display()); + let available = self + .remaining + .saturating_sub(if self.out.blocks.is_empty() { 0 } else { 2 }); + if available < label.len() { + return self.omit(file); + } + let limit = self.settings.max_bytes.min(available - label.len()); + let body = bounded_text(&definition.body, limit); + definition.verify()?; + if let Some(body) = self.record_body(file, body) { + self.expand_rule_imports(path, root, project, &body)?; + } + Ok(()) + } + + fn expand_rule_imports( + &mut self, + path: &Path, + root: &Path, + project: bool, + body: &Bounded, + ) -> Result<(), String> { + self.stack.push(path.into()); + for reference in imports(body.complete_lines()) { + self.import(path, &reference, project.then_some(root)); + } + self.stack.pop(); + Ok(()) + } + /// Load a candidate file. A missing file is skipped; a file that resolves outside /// `confine` (the canonical project root of a project file) or that the read gate /// refuses is skipped with a warning; any other problem, a path that is not a regular @@ -420,15 +682,16 @@ impl<'a> Loader<'a> { /// Read and render one file whose canonical path is `key`, then its imports. fn include(&mut self, path: &Path, key: PathBuf, confine: Option<&Path>) -> Result<(), String> { - if !self.seen.insert(key.clone()) { + if !self.seen.insert(key.display().to_string()) { return Ok(()); } - if self.delivered.contains(&key) { + if self.delivered.contains(&key.display().to_string()) { return self.discover_imports(path, key, confine); } let file = InstructionFile { path: path.to_path_buf(), canonical: key.clone(), + rule_binding: None, truncated: false, }; let separator = if self.out.blocks.is_empty() { 0 } else { 2 }; @@ -444,9 +707,27 @@ impl<'a> Loader<'a> { .min(self.discovery_remaining); let read = read_bounded(path, limit)?; self.discovery_remaining -= read.text.len(); + let Some(body) = self.record_body(file, read) else { + return Ok(()); + }; + let references = imports(body.complete_lines()); + self.stack.push(key); + for reference in references { + self.import(path, &reference, confine); + } + self.stack.pop(); + Ok(()) + } + + fn record_body(&mut self, file: InstructionFile, read: Bounded) -> Option { + let path = file.path.clone(); + let path = path.as_path(); + let separator = if self.out.blocks.is_empty() { 0 } else { 2 }; + let available = self.remaining.saturating_sub(separator); let Some(rendered) = assembly::block(path, &read, self.settings.max_bytes, available) else { - return self.omit(file); + let _ = self.omit(file); + return None; }; self.remaining -= rendered.text.len() + separator; let file = InstructionFile { @@ -469,18 +750,13 @@ impl<'a> Loader<'a> { ) }); } - let references = imports(rendered.body.complete_lines()); + let body = rendered.body.clone(); self.out.files.push(file.clone()); self.out.blocks.push(assembly::Block { file, body: rendered.body, }); - self.stack.push(key); - for reference in references { - self.import(path, &reference, confine); - } - self.stack.pop(); - Ok(()) + Some(body) } /// Revisit authorized import declarations without redelivering an already labelled @@ -543,11 +819,36 @@ impl<'a> Loader<'a> { "is not allowed by the read permission rules and was not imported", ); } - if let Err(e) = self.include(&target, key, confine) { + let result = if self.is_rule(&key) { + let root = self.rule_root.clone(); + let target_file = self.rule_target.clone(); + let global = key.parent() == Some(self.global_rules.as_path()); + self.rule(&key, &root, target_file.as_deref(), !global) + } else { + self.include(&target, key, confine) + }; + if let Err(e) = result { self.warn(from, reference, &format!("could not be read: {e}")); } } + fn is_rule(&self, path: &Path) -> bool { + let Some(parent) = path.parent() else { + return false; + }; + parent == self.global_rules + || (parent + .file_name() + .is_some_and(|name| name.to_string_lossy().eq_ignore_ascii_case("rules")) + && parent + .parent() + .and_then(Path::file_name) + .is_some_and(|name| { + name.to_string_lossy().eq_ignore_ascii_case(".cyber") + || name.to_string_lossy().eq_ignore_ascii_case(".claude") + })) + } + fn resolve(&self, from: &Path, reference: &str) -> Option { if let Some(rest) = reference.strip_prefix("~/") { return self.settings.home.as_ref().map(|home| home.join(rest)); @@ -574,6 +875,18 @@ struct Bounded { limit: usize, } +fn bounded_text(text: &str, limit: usize) -> Bounded { + let mut end = text.len().min(limit); + while !text.is_char_boundary(end) { + end -= 1; + } + Bounded { + text: text[..end].into(), + omitted: text.len() - end, + limit, + } +} + impl Bounded { /// A cut final line is not an import declaration: its truncated path could name /// a different existing file. Full final lines without a newline remain valid. @@ -948,11 +1261,7 @@ mod tests { "{}\nweb cyber\n", web_cyber.display() ))); - let attached = nested - .files - .iter() - .map(|file| file.canonical.display().to_string()) - .collect(); + let attached = nested.files.iter().map(|file| file.identity()).collect(); let again = load_nested(&repo.global, &repo.root, &file, &settings(), &attached); assert!(again.files.is_empty()); let outside = load_nested( @@ -1153,10 +1462,123 @@ mod tests { let before = attached.len(); for file in delivery.files { assert!(delivery.text.contains(&file.path.display().to_string())); - attached.insert(file.canonical.display().to_string()); + attached.insert(file.identity()); } assert!(attached.len() > before); } assert_eq!(attached.len(), 21, "{attached:?}"); } + + #[test] + fn global_scoped_rules_survive_project_opt_out_without_loading_project_rules() { + let repo = Repo::new(); + std::fs::create_dir(repo.global.join("rules")).unwrap(); + std::fs::write( + repo.global.join("rules/web.md"), + "---\npaths: ['web/**']\n---\nGLOBAL_SCOPED_RULE", + ) + .unwrap(); + repo.write(".cyber/rules/web.md", "PROJECT_RULE"); + let target = repo.write("web/app.ts", ""); + let mut settings = settings(); + settings.project = false; + assert!(repo.baseline("", &settings).text().is_none()); + let nested = load_nested( + &repo.global, + &repo.root, + &target, + &settings, + &BTreeSet::new(), + ); + let text = nested.reminder().unwrap(); + assert!(text.contains("GLOBAL_SCOPED_RULE")); + assert!(!text.contains("PROJECT_RULE")); + } + + #[test] + fn approval_snapshot_cannot_authorize_changed_rule_bytes() { + let repo = Repo::new(); + let path = repo.write(".cyber/rules/a.md", "APPROVED_RULE"); + let digests = cyber_core::rules::trust_subset(&repo.root).digests; + let root = repo.root.clone(); + let mut settings = settings(); + settings.rule_gate = Some(RuleGate::new(move |path, _| { + Ok(digests.get(path).map(|digest| RuleAuthorization { + root: root.clone(), + digest: digest.clone(), + generation: "reviewed".into(), + })) + })); + assert!( + repo.baseline("", &settings) + .text() + .unwrap() + .contains("APPROVED_RULE") + ); + std::fs::write(path, "CHANGED_RULE").unwrap(); + let changed = repo.baseline("", &settings); + assert!(changed.text().is_none()); + assert!( + changed + .warnings + .iter() + .any(|warning| warning.contains("differs from its approved digest")) + ); + } + + #[cfg(unix)] + #[test] + fn an_explicit_global_root_alias_cannot_turn_a_scoped_import_into_baseline_text() { + let repo = Repo::new(); + std::fs::create_dir(repo.global.join("rules")).unwrap(); + std::fs::write( + repo.global.join("rules/web.md"), + "---\npaths: ['web/**']\n---\nSCOPED_GLOBAL_GUIDANCE", + ) + .unwrap(); + std::fs::write(repo.global.join("CYBER.md"), "@rules/web.md").unwrap(); + let alias = repo.global.parent().unwrap().join("global-alias"); + std::os::unix::fs::symlink(&repo.global, &alias).unwrap(); + let baseline = load_baseline(&alias, &repo.root, &settings()).unwrap(); + assert!(!baseline.text().unwrap().contains("SCOPED_GLOBAL_GUIDANCE")); + let target = repo.write("web/app.ts", ""); + let nested = load_nested(&alias, &repo.root, &target, &settings(), &BTreeSet::new()); + assert!( + nested + .reminder() + .unwrap() + .contains("SCOPED_GLOBAL_GUIDANCE") + ); + } + + #[test] + fn imported_definitions_are_not_charged_again_when_the_directory_lists_them() { + let repo = Repo::new(); + repo.write(".cyber/rules/a.md", "@b.md\n@c.md\n@d.md"); + for name in ["b", "c", "d"] { + repo.write(&format!(".cyber/rules/{name}.md"), &"x".repeat(1024 * 1024)); + } + let digests = cyber_core::rules::trust_subset(&repo.root).digests; + let root = repo.root.clone(); + let mut settings = settings(); + settings.rule_gate = Some(RuleGate::new(move |path, _| { + Ok(digests.get(path).map(|digest| RuleAuthorization { + root: root.clone(), + digest: digest.clone(), + generation: "reviewed".into(), + })) + })); + let loaded = repo.baseline("", &settings); + assert_eq!(loaded.files.len(), 4); + assert!(loaded.text().unwrap().len() <= DEFAULT_MAX_TOTAL_BYTES); + assert!( + !loaded + .warnings + .iter() + .any(|warning| warning.contains("rule discovery bound") + || warning.contains("rule definition byte bound")), + "{:?}", + loaded.warnings + ); + } } diff --git a/crates/cyber-server/src/runtime/instructions/assembly.rs b/crates/cyber-server/src/runtime/instructions/assembly.rs index 09f90a034..f8624b8f7 100644 --- a/crates/cyber-server/src/runtime/instructions/assembly.rs +++ b/crates/cyber-server/src/runtime/instructions/assembly.rs @@ -122,7 +122,7 @@ pub(super) fn finish(text: String, files: Vec) -> Option, + /// Opaque identities of rule definitions delivered this Epoch. + #[serde(default, skip_serializing_if = "std::collections::BTreeSet::is_empty")] + pub reminded_rules: std::collections::BTreeSet, pub number: u32, pub baseline: String, pub snapshot: BTreeMap, @@ -1384,6 +1387,7 @@ impl SessionState { fn on_epoch(&mut self, e: EpochStarted) { self.epoch = Some(Epoch { reminded_skills: Default::default(), + reminded_rules: Default::default(), reminded_instructions: Default::default(), number: e.epoch, baseline: e.baseline, @@ -1504,6 +1508,13 @@ impl SessionState { { epoch.reminded_skills.extend(s.skill_reminders); epoch.reminded_instructions.extend(s.instruction_reminders); + if s.status == CallStatus::Ok { + epoch.reminded_rules.extend( + s.rule_reminders + .iter() + .map(super::instructions::RuleBinding::identity), + ); + } } call.skill_activation = if s.status == CallStatus::Ok { s.skill_activation diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index dcdcb9fa9..8198c8e12 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1145,6 +1145,39 @@ impl ToolHost for BuiltinHost { cyber_core::config::InstructionLimits::from_config(&config) } + fn instruction_rule_gate( + &self, + directory: &str, + ) -> Option { + use cyber_server::runtime::instructions::{RuleAuthorization, RuleGate}; + let resolve = self.hook_config.get()?.resolve.clone(); + let root = cyber_core::config::project_root(Path::new(directory)) + .canonicalize() + .ok()?; + // A discovery may contain many files in one config directory. Resolve its + // trust inventory once for this observation, never reuse it across calls. + let cache = Mutex::new(BTreeMap::< + PathBuf, + Result, + >::new()); + Some(RuleGate::new(move |path, directory| { + if !directory.starts_with(&root) || !path.starts_with(&root) { + return Ok(None); + } + let mut cache = cache + .lock() + .map_err(|_| "rule trust cache unavailable".to_owned())?; + let report = cache + .entry(directory.into()) + .or_insert_with(|| resolve(directory).map(|config| config.trust)); + let report = report.as_ref().map_err(Clone::clone)?; + if report.checkout_root != root { + return Ok(None); + } + Ok(RuleAuthorization::from_report(report, path)) + })) + } + fn finalize_skill_output( &self, directory: &str, diff --git a/crates/cyber-tools/tests/path_skills.rs b/crates/cyber-tools/tests/path_skills.rs index aadd34575..0e31fd0d5 100644 --- a/crates/cyber-tools/tests/path_skills.rs +++ b/crates/cyber-tools/tests/path_skills.rs @@ -345,6 +345,7 @@ async fn post_hook_failure_keeps_the_mutation_failure_and_durable_reminder() { plugins: Vec::new(), pure: false, trust: TrustReport { + rule_digests: Default::default(), checkout_root: root.clone(), trusted: true, digest: None, diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index d080d6f85..2c0ae1828 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Complete trusted path-rule discovery/precedence and authorized-operation reminders with Epoch provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Finish edit/patch/partial-effect delivery, exact operation/config/Epoch and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 0fb4d161d..a9d563dda 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1201,6 +1201,28 @@ Files in directories below the Location are not in the baseline. When the built- One complete reminder, including its wrapper, labels, notices and file list, fits the smaller of 32768 bytes and `instruction_limits.max_total_bytes`. Files beyond the content budget are named for an explicit read when their complete labels fit. Only files actually labelled in the delivered reminder count as attached; unlisted files remain eligible. Authorized import declarations are revisited under a bounded discovery budget so naming an importer does not strand its unlisted children. A cut final import line is ignored rather than interpreted as another filename. When even that reminder does not fit the tool output budget (`tool_output`), the read gets a short list of the file paths; when nothing fits, the read keeps its own output and the files can attach on a later read. Instruction files never turn a successful read into an error. Each attachment fires the `InstructionsLoaded` hook with `load_reason: "nested_read"` and the attached `file_paths`, as the baseline does with `session_start` and `changed`. +### Path-scoped rule files + +Cyber Code discovers sorted Markdown files in `.claude/rules/` and then `.cyber/rules/` at each directory from the project root toward the Location or successfully read target. Rules follow the ordinary instruction files of their directory; `AGENTS.override.md` keeps its existing replacement behavior. Owner-global rules live in the config directory's `rules/` folder. `compat.claude = false` and the Claude compatibility environment switches suppress Claude rules, including imports of them. + +A rule without `paths` joins the instruction baseline. A scoped rule uses optional YAML frontmatter: + +```markdown +--- +paths: + - "web/src/**" +--- +Keep components accessible and test keyboard navigation. +``` + +Patterns are relative to the project root, with `/` separators. Invalid, empty, absolute, parent-traversing or oversized patterns are rejected rather than treated as unscoped guidance. A rule accepts only `paths` and optional `description`; frontmatter cannot configure tools or permissions. Ordinary `@path` import rules still apply, and importing another scoped rule does not bypass its path filter or review/read gates. + +Project rule bytes are included in the existing checkout trust digest. Review with `cyber trust inspect`, then approve that exact digest with `cyber trust approve --digest `. For definitions in a nested config directory, inspect at that directory (`cyber --cwd web trust inspect`). The loader resolves the operation directory's reviewed inventory and accepts definitions only from the same checkout. Changed bytes require renewed review. Approvals retain the existing checkout-digest storage semantics; this adds no new trust-policy format. + +Definition reads use the fenced native reader: linked files/directories, hard links and special files are refused. The reader caps a definition at 1 MiB; frontmatter is limited to 16 KiB, with up to 64 relative patterns of 256 bytes each. A project inventory covers up to 128 definitions and 4 MiB, with bounded directory enumeration. These review/discovery bounds are separate from the smaller rendered instruction budgets. Unknown inventories grant no rule authority. `cyber doctor` reports held, refused and truncated baseline rules. + +The current local implementation activates scoped rules after successful built-in reads and records typed root/path/digest/approval bindings separately from legacy instruction paths. Edit/patch and partial-settlement delivery, exact operation/Epoch binding, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. + ### Switches These are read from the server's environment (a value of `1`, `true`, `yes` or `on`): diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 837814706..35f97d6b7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -22,6 +22,10 @@ Use the canonical 200 MiB per-Location fallback limit and seven-day default rete Load `.cyber/rules/*.md` and enabled `.claude/rules/*.md` at the existing directory precedence. Rules without `paths` join the baseline. Relative `paths` globs activate after a successful authorized read/edit/patch of a matching path; denied/failed operations do not consume once-per-Epoch delivery. Successful partial file settlements retain correct reminder ownership. Bind project root, definition digest, trust/config generation, successful operation identity and Context Epoch. Keep existing directory instruction ordering and `AGENTS.override.md` replacement semantics; do not use a project rule to widen tool/permission authority. +Rule definitions use the existing fenced native source reader (no linked/hard-linked/special sources), with full definition bytes represented by SHA-256 in the existing checkout trust inventory. The reviewed manifest remains private configuration metadata. Project loading requires a trusted expected digest, same checkout root and the Location's read gate before definition capture; captured bytes and retained native bindings are verified before rendering. The operation directory's ancestry selects nested definitions using existing checkout-digest approval semantics, without changing #92's pending trust-key policies. Global owner rules use `/rules`. Directory text order is ordinary instruction files, sorted Claude rules when enabled, then sorted Cyber rules, with root directories before more specific ones. Imports retain existing confinement/read/depth/cycle checks and recognize scoped rule targets instead of loading their frontmatter as unconditional text. + +Definition discovery bounds are 128 project files, 4096 examined entries per rules directory, 1 MiB per fenced definition and 4 MiB per inventory/loader; frontmatter is capped at 16 KiB and 64 portable relative globs of at most 256 bytes. Invalid/null/empty `paths` and unknown authority-bearing keys are errors, never baseline fallbacks. Rule identity includes root, definition path, exact bytes and approval generation. Successful delivery stores optional typed `rule_reminders` separately from the legacy canonical `instruction_reminders`; historical events default to no rule bindings. Exact native operation/config-generation/Epoch binding and withdrawal/restart behavior remain required under 4.2–4.4. + Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. ### 4. Root structured output is per admitted prompt diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index fd0bd03ba..8634ac7a3 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -41,3 +41,16 @@ - **THEN** the complete rendered instruction-file baseline (`core/instructions`) and each nested reminder SHALL fit their respective byte budgets, including labels, notices, separators and wrappers - **AND** only complete file labels actually delivered SHALL consume reminder identity; unlisted imports SHALL remain discoverable after their importer is labelled - **AND** a truncated import declaration SHALL NOT load the file named by its partial path + +#### Scenario: Rule review and byte identity +- **WHEN** project rule files are discovered +- **THEN** their exact bounded definition bytes SHALL enter the checkout trust inventory, and held or read-denied definitions SHALL NOT enter privileged model context +- **AND** invalid frontmatter/globs SHALL NOT fall back to unscoped guidance +- **WHEN** a definition changes and is reviewed again +- **THEN** its new content/approval identity SHALL remain independently eligible for delivery + +#### Scenario: Compatible durable rule bindings +- **WHEN** a successful matching read delivers a scoped rule +- **THEN** its settlement SHALL retain typed project-root, definition-path, digest and approval-generation metadata separately from canonical instruction-file path markers +- **AND** replay of historical settlements without rule bindings SHALL remain compatible +- **AND** a reviewed definition from another checkout SHALL NOT supply rule context for the parent Session diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 888da3dd7..9761bee20 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1621,3 +1621,60 @@ rediscovery and partial-line refusal. Full rules/context review remains open for trusted definition identity, source-graph exhaustion, operation/Epoch/restart and revocation binding, effective inspection and the TUI flow. Task 4.1 and parent acceptance remain incomplete; no goal or delivery completion is claimed. + +Trusted rule discovery/read continuation (2026-11-10): global `rules/` and +project `.claude/rules/*.md` / `.cyber/rules/*.md` now use the fenced native +definition reader, sorted directory precedence and validated project-relative +globs. Unscoped bodies join the baseline; matching scoped bodies enter successful +built-in read results. Claude opt-out and project-config opt-out are respected. +Frontmatter is not rendered or used to configure permissions. Imports retain +confinement/read/depth/cycle checks and recognize rule targets, so importing a +scoped rule cannot turn it into unconditional instruction text. + +Exact bounded rule-file bytes enter the existing checkout trust digest. Loading +requires an approved expected digest and the original Location's read gate before +project definition capture, followed by fenced byte/binding verification. Nested +definitions resolve the operation directory's reviewed inventory in the same +checkout; a foreign nested checkout cannot contribute rules. Changed bytes remain +held until renewed review. No #92 trust-key policy or approval-file format changes. +Trust inventory paths are canonicalized at the declared checkout root, preserving +ordinary caller aliases while retaining no-follow checks below it. Explicit global +root aliases cannot bypass scoped-import classification. + +Read delivery now persists optional typed `rule_reminders` (root, definition path, +byte digest and approval generation), separate from legacy canonical instruction +paths. The Epoch tracks rule identities separately; historical events default to +no rule bindings. Actual App Store-event inspection and replay verify this +separation. Directory/import duplicates are captured/charged only once per +discovery, preventing a valid near-limit graph from spuriously exhausting its +definition budget. Full operation/config-generation/Epoch binding, edit/patch +and partial-effect delivery, revocation withdrawal, graph-exhaustion and restart +acceptance remain required under 4.2–4.4. No parent task is checked off. + +Verification passes: five selected core library tests (four rule tests and the +existing untrusted-rule table check); all 89 configuration tests; 454 server +library tests with one explicit subprocess-fixture entry ignored; all 15 real-App +instruction tests; 20 context/compaction tests; AppHost forwarding; nine path-skill +compatibility tests; and the doctor regression. App tests cover withheld/approved +rules, baseline/read separation, once-per-Epoch identity, renewed review after +edits, nested/foreign roots, read-denied definitions, Claude opt-out, imports, +bounded provider text and typed durable bindings. Native link/hardlink refusal, +invalid/null/empty/escaping glob refusal and inventory bounds have core coverage. +OpenAPI's exact consistency test and SDK generation check pass; no HTTP schema or +generated SDK change is required. Full workspace/SDK runtime and fresh-source +platform acceptance have not run for this continuation. + +Focused code-review-skill review resolved empty-array/null fallback risks, alias +classification, duplicate budget charging and mixing opaque rule IDs into the +legacy path field. Architecture follows the existing core fenced reader and +runtime ToolHost authorization port. Full rule/context review remains open for +exact native operation/config/Epoch revalidation and provenance, withdrawal after +revocation, graph/restart boundaries and effective inspection/TUI. Native C and +linker unwind warnings were observed without assigning origin. The focused native +workflow now includes core rule and trust tests plus skill compatibility checks. +Task 4.1 and all parent acceptance/delivery obligations remain open. + +Final local gates: core/server/tools/App/CLI all-target Clippy with warnings denied, +formatting, diff checks and workflow YAML/source-selection checks pass. Strict +OpenSpec 1.13.2 validates 100/100 items; cross-spec lint reports 0 errors/25 +warnings. The worktree must execute on fresh-source CI before platform acceptance. From fa2c9dfb370e440eb168a647f77f4b64fa54111a Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 16:04:01 -0400 Subject: [PATCH 77/87] Fence nested reminders to the active context epoch --- .../cyber-server/src/runtime/auto_override.rs | 1 + crates/cyber-server/src/runtime/drain.rs | 165 ++++++++++++++++-- .../cyber-server/tests/context_compaction.rs | 57 ++++++ docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 6 +- .../complete-daily-coding-surfaces/design.md | 2 + .../specs/system-context/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 28 +++ 8 files changed, 249 insertions(+), 19 deletions(-) diff --git a/crates/cyber-server/src/runtime/auto_override.rs b/crates/cyber-server/src/runtime/auto_override.rs index 725e3be77..2601b99b1 100644 --- a/crates/cyber-server/src/runtime/auto_override.rs +++ b/crates/cyber-server/src/runtime/auto_override.rs @@ -473,6 +473,7 @@ impl Inner { .await; let nested = nested.pop().unwrap_or_default(); let mut state = handle.state.lock().await; + let nested = run.current_nested(&state, nested); let mut seen = super::drain::Reminded::of(&state); let (settled, loaded) = super::drain::settlement_with_reminders( &replay.id, diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index e37b61a9e..6dadc68a0 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -1819,6 +1819,7 @@ impl Inner { let mut stops = Vec::new(); let mut loaded = Vec::new(); for (run, nested) in outcomes.into_iter().zip(nested) { + let nested = run.current_nested(&state, nested); let (settled, instructions) = settlement_with_reminders( &run.call_id, &run.def, @@ -2030,7 +2031,8 @@ impl Inner { cancel: CancellationToken, ) -> ToolRun { let call_id = invocation.call_id.clone(); - let read = self.read_target(&def, &invocation); + let scope = ReminderScope::capture(&*handle.state.lock().await, &invocation.directory); + let read = scope.and_then(|scope| self.read_target(&def, &invocation, scope)); if invocation.name == super::structured::CHILD_TOOL || invocation.name == super::structured::ROOT_TOOL { @@ -2069,7 +2071,12 @@ impl Inner { } /// The file a built-in `read` call targets, for nested instructions. - fn read_target(&self, def: &ToolDef, invocation: &Invocation) -> Option { + fn read_target( + &self, + def: &ToolDef, + invocation: &Invocation, + scope: ReminderScope, + ) -> Option { if def.scope != super::host::ToolScope::Builtin || def.spec.name != "read" { return None; } @@ -2079,7 +2086,7 @@ impl Inner { Some(ReadTarget { file: super::instructions::read_target(&location, settings.home.as_deref(), path), location, - settings, + scope, }) } @@ -2098,17 +2105,29 @@ impl Inner { if targets.iter().all(Option::is_none) { return empty(); } - let attached = Reminded::of(&*handle.state.lock().await).instructions; + let state = handle.state.lock().await; + let targets: Vec<_> = targets + .into_iter() + .map(|target| target.filter(|read| read.scope.is_current(&state))) + .collect(); + let attached = Reminded::of(&state).instructions; + drop(state); + let inner = self.me.upgrade().expect("runtime alive"); let global = self.options.global_config_dir.clone(); let load = move || { let mut loaded = Vec::with_capacity(targets.len()); for target in targets { let nested = target.map_or_else(Default::default, |read| { + // Resolve again after the effect: pre-call permission callbacks and + // trust inventories may have been invalidated while the tool ran. + let Ok(settings) = inner.instruction_settings(&read.scope.directory) else { + return Default::default(); + }; super::instructions::load_nested( &global, &read.location, &read.file, - &read.settings, + &settings, &attached, ) }); @@ -2336,6 +2355,24 @@ pub(super) struct ToolRun { } impl ToolRun { + /// Recheck under the settlement lock, after filesystem discovery. A Location + /// switch or Epoch replacement during discovery must not consume new identities. + pub(super) fn current_nested( + &self, + state: &SessionState, + nested: super::instructions::Instructions, + ) -> super::instructions::Instructions { + if self + .read + .as_ref() + .is_some_and(|read| read.scope.is_current(state)) + { + nested + } else { + Default::default() + } + } + /// The target to load nested instructions for: only a read that succeeded. pub(super) fn nested_target(&self) -> Option { let succeeded = matches!( @@ -2348,12 +2385,41 @@ impl ToolRun { } } -/// A `read` call's Location, the file it targets and what may load there. +/// The context in which the successful operation may deliver a reminder. +#[derive(Clone)] +pub(super) struct ReminderScope { + directory: String, + epoch: u32, +} + +impl ReminderScope { + fn capture(state: &SessionState, directory: &str) -> Option { + if state.epoch_stale || state.info.directory != directory { + return None; + } + Some(Self { + directory: directory.into(), + epoch: state.epoch.as_ref()?.number, + }) + } + + fn is_current(&self, state: &SessionState) -> bool { + !state.epoch_stale + && state.info.directory == self.directory + && state + .epoch + .as_ref() + .is_some_and(|epoch| epoch.number == self.epoch) + } +} + +/// A `read` call's Location, target and captured context; settings are resolved +/// after successful execution rather than retaining pre-effect permission closures. #[derive(Clone)] pub(super) struct ReadTarget { pub location: std::path::PathBuf, pub file: std::path::PathBuf, - pub settings: super::instructions::InstructionSettings, + pub scope: ReminderScope, } /// `InstructionsLoaded` `load_reason` for files a read attached. @@ -2736,6 +2802,75 @@ mod skill_reminder_tests { use super::super::host::{SkillSuggestion, ToolScope}; use super::*; + #[test] + fn reminders_cannot_cross_an_epoch_or_location_change_during_loading() { + let state = || { + let info = serde_json::from_value(serde_json::json!({ + "id":"ses_rules", "title":"rules", "directory":"/repo", "parent_id":null, + "agent":"build", "model":"test/main", "mode":"default", "created_ms":0 + })) + .unwrap(); + let mut state = SessionState::new(info); + state.epoch = Some( + serde_json::from_value(serde_json::json!({ + "number":1, "baseline":"", "snapshot":{}, "provider":"test" + })) + .unwrap(), + ); + state + }; + for changed in ["epoch", "location", "stale", "missing"] { + let mut state = state(); + let scope = ReminderScope::capture(&state, "/repo").unwrap(); + let run = ToolRun { + call_id: "read".into(), + def: read_def(), + outcome: ToolOutcome::Ok("file".into()), + read: Some(ReadTarget { + location: "/repo".into(), + file: "/repo/web/app.ts".into(), + scope, + }), + }; + match changed { + "epoch" => state.epoch.as_mut().unwrap().number += 1, + "location" => state.info.directory = "/other".into(), + "stale" => state.epoch_stale = true, + _ => state.epoch = None, + } + let (_temp, candidate) = nested(&[("CYBER.md", "nested delivery".into())]); + let held = run.current_nested(&state, candidate); + assert!(held.is_empty(), "{changed}"); + let mut seen = Reminded::of(&state); + let (settled, loaded) = settle(&held, &mut seen, default_budget); + assert!(loaded.is_none(), "{changed}"); + assert!(settled.instruction_reminders.is_empty(), "{changed}"); + assert!(seen.instructions.is_empty(), "{changed}"); + // A new operation in the active context can still deliver the identity. + state.epoch_stale = false; + if state.epoch.is_some() { + let scope = ReminderScope::capture(&state, &state.info.directory).unwrap(); + let fresh = ToolRun { + read: Some(ReadTarget { + location: state.info.directory.clone().into(), + file: "/repo/web/app.ts".into(), + scope, + }), + ..run + }; + let (_temp, candidate) = nested(&[("CYBER.md", "nested delivery".into())]); + let current = fresh.current_nested(&state, candidate); + let (settled, loaded) = settle(¤t, &mut seen, default_budget); + assert!(loaded.is_some()); + assert_eq!(settled.instruction_reminders.len(), 1); + } + } + let mut initial = state(); + initial.epoch = None; + assert!(ReminderScope::capture(&initial, "/repo").is_none()); + assert!(ReminderScope::capture(&state(), "/other").is_none()); + } + #[test] fn legacy_settlements_keep_canonical_paths_and_have_no_fabricated_rule_bindings() { let settled: ToolSettled = serde_json::from_value(serde_json::json!({"call_id":"old","status":"ok","output":"text","instruction_reminders":["/repo/web/CYBER.md"]})).unwrap(); @@ -2933,10 +3068,10 @@ mod skill_reminder_tests { let read = ReadTarget { location: "/repo".into(), file: "/repo/web/app.ts".into(), - settings: super::super::instructions::InstructionSettings::from_env( - true, - &std::collections::HashMap::::new(), - ), + scope: ReminderScope { + directory: "/repo".into(), + epoch: 1, + }, }; let run = |outcome| ToolRun { call_id: "call".into(), @@ -3009,10 +3144,10 @@ mod skill_reminder_tests { read: Some(ReadTarget { location: "/repo".into(), file: "/repo/a.md".into(), - settings: super::super::instructions::InstructionSettings::from_env( - true, - &std::collections::HashMap::::new(), - ), + scope: ReminderScope { + directory: "/repo".into(), + epoch: 1, + }, }), }; assert!(run.nested_target().is_none()); diff --git a/crates/cyber-server/tests/context_compaction.rs b/crates/cyber-server/tests/context_compaction.rs index 9a0443cf1..de9ac5a6f 100644 --- a/crates/cyber-server/tests/context_compaction.rs +++ b/crates/cyber-server/tests/context_compaction.rs @@ -675,3 +675,60 @@ async fn compaction_summary_carries_no_opaque_state_and_the_tail_replays() { .collect(); assert_eq!(tail, [("new thought", true)]); } + +/// A successful operation does not authorize instruction loading with callbacks +/// captured before a configuration change while the operation was in flight. +#[tokio::test] +async fn nested_instructions_resolve_settings_after_the_successful_operation() { + let h = Harness::new(Setup { + scripts: vec![( + "test/main", + vec![ + tools(&[("read_one", "read", r#"{"path":"web/app.ts"}"#)]), + text("done"), + tools(&[("read_two", "read", r#"{"path":"web/app.ts"}"#)]), + text("done"), + ], + )], + ..Setup::default() + }); + std::fs::create_dir(h.repo.join("web")).unwrap(); + std::fs::write(h.repo.join("web/CYBER.md"), "fresh operation instructions").unwrap(); + std::fs::write(h.repo.join("web/app.ts"), "contents").unwrap(); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::Gated); + let id = h.session().await; + h.runtime.admit(&id, admit("first read")).await.unwrap(); + tokio::time::timeout( + std::time::Duration::from_secs(5), + h.tools.started.notified(), + ) + .await + .unwrap(); + *h.tools.instruction_limits.lock().unwrap() = Err("configuration changed during read".into()); + h.tools.release.notify_one(); + h.settle(&id).await; + let state = h.state(&id).await; + assert_eq!(state.calls["read_one"].output.as_deref(), Some("done")); + assert!(state.epoch.unwrap().reminded_instructions.is_empty()); + *h.tools.instruction_limits.lock().unwrap() = Ok(Default::default()); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::Return("contents".into())); + h.runtime.admit(&id, admit("second read")).await.unwrap(); + h.settle(&id).await; + let state = h.state(&id).await; + assert!( + state.calls["read_two"] + .output + .as_deref() + .unwrap() + .contains("fresh operation instructions") + ); + assert_eq!(state.epoch.unwrap().reminded_instructions.len(), 1); +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 2c0ae1828..1da0bab56 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Finish edit/patch/partial-effect delivery, exact operation/config/Epoch and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Nested read delivery now reloads settings after success and checks captured Location/Epoch before discovery and under the settlement lock. Finish edit/patch/partial-effect delivery, native operation/config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index a9d563dda..d32aa829b 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1221,7 +1221,7 @@ Project rule bytes are included in the existing checkout trust digest. Review wi Definition reads use the fenced native reader: linked files/directories, hard links and special files are refused. The reader caps a definition at 1 MiB; frontmatter is limited to 16 KiB, with up to 64 relative patterns of 256 bytes each. A project inventory covers up to 128 definitions and 4 MiB, with bounded directory enumeration. These review/discovery bounds are separate from the smaller rendered instruction budgets. Unknown inventories grant no rule authority. `cyber doctor` reports held, refused and truncated baseline rules. -The current local implementation activates scoped rules after successful built-in reads and records typed root/path/digest/approval bindings separately from legacy instruction paths. Edit/patch and partial-settlement delivery, exact operation/Epoch binding, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. +The current local implementation activates scoped rules after successful built-in reads and records typed root/path/digest/approval bindings separately from legacy instruction paths. Nested discovery resolves settings after the successful read and checks that its captured Location and Context Epoch still apply before loading and before settlement. A suppressed reminder remains eligible for a later operation. Edit/patch and partial-settlement delivery, native operation and exact configuration-generation binding, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. ### Switches @@ -1240,7 +1240,7 @@ These are read from the server's environment (a value of `1`, `true`, `yes` or ` No migration is needed: `CLAUDE.md` keeps loading in place. `cyber import` does not copy `CLAUDE.md` into `CYBER.md`, because both would then load and duplicate the content. To move Cyber-specific parts, put them in `CYBER.md` and, if the remaining guidance is shared, rename `CLAUDE.md` to `AGENTS.md`. -Not implemented yet: `.cyber/rules/*.md` and `.claude/rules/*.md`, `.claude/CLAUDE.md`, the `~/.claude/CLAUDE.md` global fallback, a configurable `instructions.max_bytes`, per-source token sizes in `/context`, and `/init` (which is specified to generate `AGENTS.md`, not `CYBER.md`). +Not implemented yet: `.claude/CLAUDE.md`, the `~/.claude/CLAUDE.md` global fallback, per-source token sizes in `/context`, and `/init` (which is specified to generate `AGENTS.md`, not `CYBER.md`). ## Base prompts and output styles @@ -2099,7 +2099,7 @@ Nothing a source declares is run during an import, and an item without a native | Claude `hooks` (every settings file, in layer order), Codex `hooks.json` | `hooks`, events one to one; `Bash` → `bash`, `Edit\|Write` → `{edit,write}`, `mcp__x__.*` → `mcp__x__*`; `statusMessage` → `status_message`; `CLAUDE_PROJECT_DIR` → `CYBER_PROJECT_DIR`. Events without an equivalent, `agent` handlers, header-carrying handlers and regex matchers are reported | | Claude `.claude/agents/*.md` | `agents.`: the body is `system`, `mode: subagent`, `tools`/`disallowedTools` → `tools.allow`/`tools.deny`, `model` (aliases as above, `inherit` omitted), `color`, `permissionMode` → `permission_mode`, `skills`. A denied tool without an equivalent refuses the agent | | Claude `apiKeyHelper` | reported: command credentials are not available yet; set `ANTHROPIC_API_KEY` | -| Claude `.claude/rules/*.md` | reported: rule files are not loaded yet | +| Claude `.claude/rules/*.md` | reported without conversion; the runtime loads reviewed definitions in place when Claude compatibility is enabled | | Skills (`.claude/skills`, `.agents/skills`, `.opencode/skills`, `~/.codex/skills`), `CLAUDE.md`, `AGENTS.md` | not copied: reported `imported` because the runtime reads them in place | | Codex `[profiles.*]`, `~/.codex/.config.toml`, `profile` | `profiles.` with `model`, `mode` and `sandbox.policy`; `default_profile` | | Codex `[permissions.*]`, `default_permissions` | `sandbox.profiles.` (`extends`, `filesystem` paths → `writable_roots`/`readable_paths`/`deny_read`, `network.enabled` → `off`/`proxy`, `network.domains`); `sandbox.profile`. A profile that cannot be converted is reported, and selecting it selects `read-only` instead | diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 35f97d6b7..741fa265d 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -26,6 +26,8 @@ Rule definitions use the existing fenced native source reader (no linked/hard-li Definition discovery bounds are 128 project files, 4096 examined entries per rules directory, 1 MiB per fenced definition and 4 MiB per inventory/loader; frontmatter is capped at 16 KiB and 64 portable relative globs of at most 256 bytes. Invalid/null/empty `paths` and unknown authority-bearing keys are errors, never baseline fallbacks. Rule identity includes root, definition path, exact bytes and approval generation. Successful delivery stores optional typed `rule_reminders` separately from the legacy canonical `instruction_reminders`; historical events default to no rule bindings. Exact native operation/config-generation/Epoch binding and withdrawal/restart behavior remain required under 4.2–4.4. +Nested read discovery resolves instruction settings after a successful effect. Its captured Session Location and Context Epoch must still be active before discovery and again under the settlement lock. An absent/stale/replaced Epoch or changed Location suppresses the reminder without recording delivery; normal group settlement and exact `/approve` replay share this check. This guards the runtime context boundary, but does not replace native operation receipts, exact configuration-generation fencing through commit, or revocation withdrawal. + Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. ### 4. Root structured output is per admitted prompt diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index 8634ac7a3..92e153614 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -54,3 +54,10 @@ - **THEN** its settlement SHALL retain typed project-root, definition-path, digest and approval-generation metadata separately from canonical instruction-file path markers - **AND** replay of historical settlements without rule bindings SHALL remain compatible - **AND** a reviewed definition from another checkout SHALL NOT supply rule context for the parent Session + +#### Scenario: Context changes while a successful read is settling +- **GIVEN** a read started in one Location and Context Epoch +- **WHEN** that Epoch becomes stale, is replaced or disappears, or the Session changes Location before reminder settlement +- **THEN** the read result SHALL settle without delivering that operation's nested reminder into the new context +- **AND** the suppressed reminder SHALL NOT consume once-per-Epoch delivery +- **AND** nested discovery SHALL resolve current instruction settings after the successful operation instead of retaining pre-effect permission callbacks diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 9761bee20..4c31c824c 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1678,3 +1678,31 @@ Final local gates: core/server/tools/App/CLI all-target Clippy with warnings den formatting, diff checks and workflow YAML/source-selection checks pass. Strict OpenSpec 1.13.2 validates 100/100 items; cross-spec lint reports 0 errors/25 warnings. The worktree must execute on fresh-source CI before platform acceptance. + + +Nested read context fence continuation (2026-11-10): capture the active Location +and Context Epoch before running the built-in read, reject absent/stale contexts, +and recheck before discovery and again under the settlement lock. Resolve current +instruction settings on the blocking discovery thread after successful execution; +retain no pre-effect permission/trust callbacks. Normal grouped settlement and +exact `/approve` replay use the same final fence. Suppression leaves successful +output intact and records no reminder identity, so a later operation can deliver. +Regressions cover replaced/missing/stale Epochs, moved Locations, and a gated +successful read whose configuration becomes unavailable before settlement, then +restored configuration and successful delivery on a later read. This does not +complete 4.2: native operation receipts, exact configuration-generation fencing +through commit, edit/patch and successful partial effects, withdrawal/revocation, +physical restart and effective inspection remain required. No parent task closes. + +Checkpoint verification: server library 455 passed / one ignored nested lock +probe; context/compaction integration 21 passed; production App instruction +integration 15 passed; HTTP override integration filter two passed. Server/App +all-target Clippy with `-D warnings`, formatting/diff checks, generated SDK +consistency and CI-pinned strict OpenSpec (100 items) passed. Spec lint reports +zero errors and 25 warnings; native C/linker warnings were observed without an +origin claim. This increment did not rerun the entire workspace, SDK runtime or +platform matrix. Checkpoint review using the code-review skill found no new +findings within these changes: existing runtime/host boundaries and atomic +settlement are retained, unavailable configuration holds reminders, both writer +paths revalidate the context, and regressions cover suppression and retry. +Whole-goal review and source-bound native acceptance remain open. From 25e7e89f68b9c79c3e2decec704acb92efda9450 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 16:19:02 -0400 Subject: [PATCH 78/87] Bind read reminders to the captured native location --- .github/workflows/non-git-snapshot-proof.yml | 2 + crates/cyber-core/src/import/mod.rs | 2 +- crates/cyber-core/src/import/snapshot.rs | 36 ++++++ crates/cyber-core/src/rules.rs | 121 ++++++++++++++++++ crates/cyber-server/src/runtime/drain.rs | 63 +++++++-- .../cyber-server/tests/context_compaction.rs | 65 ++++++++++ docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 2 +- .../complete-daily-coding-surfaces/design.md | 2 +- .../specs/system-context/spec.md | 7 + .../complete-daily-coding-surfaces/tasks.md | 46 +++++++ 11 files changed, 331 insertions(+), 17 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index ee7febbb7..724f603b8 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -28,6 +28,8 @@ on: - crates/cyber-app/tests/instructions.rs - crates/cyber-core/src/config/** - crates/cyber-core/src/rules.rs + - crates/cyber-core/src/import/snapshot.rs + - crates/cyber-core/src/import/mod.rs - crates/cyber-core/src/lib.rs - crates/cyber-core/tests/config.rs - crates/cyber-tools/src/host.rs diff --git a/crates/cyber-core/src/import/mod.rs b/crates/cyber-core/src/import/mod.rs index 3755408f0..c1c49250b 100644 --- a/crates/cyber-core/src/import/mod.rs +++ b/crates/cyber-core/src/import/mod.rs @@ -45,7 +45,7 @@ pub use preview::{ }; pub use provenance::SourceReference; pub use snapshot::SourceSnapshot; -pub(crate) use snapshot::{command_directory, verify_command_directory}; +pub(crate) use snapshot::{DirectorySnapshot, command_directory, verify_command_directory}; mod opencode_providers; pub use opencode_providers::{OpenCodeProviderConfig, opencode_provider_config}; diff --git a/crates/cyber-core/src/import/snapshot.rs b/crates/cyber-core/src/import/snapshot.rs index 6a7fb39ad..687ec5801 100644 --- a/crates/cyber-core/src/import/snapshot.rs +++ b/crates/cyber-core/src/import/snapshot.rs @@ -116,6 +116,42 @@ fn directory_chain(path: &Path) -> Result<(Vec, Vec), DiscoveryEr Ok((directories, bindings)) } +/// Native identities of every directory from the volume root to one physical +/// directory. Capturing this reads metadata only, never directory contents. +pub(crate) struct DirectorySnapshot { + path: PathBuf, + bindings: Vec, + _retained: Option>, +} +impl DirectorySnapshot { + pub(crate) fn capture(path: &Path) -> Result { + let (directories, bindings) = directory_chain(path)?; + // NTFS directory handles would freeze ordinary editor/worktree renames. + // Full Windows file IDs retain reuse protection without open handles. + #[cfg(windows)] + let retained = { + drop(directories); + None + }; + #[cfg(not(windows))] + let retained = Some(directories); + let snapshot = Self { + path: path.into(), + bindings, + _retained: retained, + }; + snapshot.verify()?; + Ok(snapshot) + } + pub(crate) fn verify(&self) -> Result<(), DiscoveryError> { + let (_, bindings) = directory_chain(&self.path)?; + if bindings != self.bindings { + return Err(error(&self.path, "source directory incarnation changed")); + } + Ok(()) + } +} + /// Internal read-time command discovery keeps every directory binding live during enumeration. pub(crate) fn command_directory(path: &Path) -> Result, DiscoveryError> { directory_chain(path).map(|(directories, _)| directories) diff --git a/crates/cyber-core/src/rules.rs b/crates/cyber-core/src/rules.rs index 8f3db4b5d..d835b89fd 100644 --- a/crates/cyber-core/src/rules.rs +++ b/crates/cyber-core/src/rules.rs @@ -11,6 +11,54 @@ pub const MAX_TOTAL_BYTES: usize = 4 * 1024 * 1024; const MAX_ENTRIES: usize = 4096; const MAX_HEADER: usize = 16 * 1024; +/// Physical checkout and Location incarnation retained across a file operation. +/// This is observation evidence, not permission to access a source or perform an effect. +pub struct RootBinding { + declared: PathBuf, + location: PathBuf, + root: PathBuf, + native: crate::import::DirectorySnapshot, +} +impl RootBinding { + pub fn capture(location: &Path) -> Result { + let declared = std::path::absolute(location).map_err(|_| "rule Location is invalid")?; + let location = location + .canonicalize() + .map_err(|_| "rule Location is unavailable")?; + let root = crate::config::project_root(&location); + let native = crate::import::DirectorySnapshot::capture(&location) + .map_err(|e| e.reason.to_owned())?; + let binding = Self { + declared, + location, + root, + native, + }; + binding.verify()?; + Ok(binding) + } + pub fn root(&self) -> &Path { + &self.root + } + pub fn verify(&self) -> Result<(), String> { + if self.declared.canonicalize().ok().as_ref() != Some(&self.location) { + return Err("declared rule Location no longer names the captured directory".into()); + } + self.native.verify().map_err(|e| e.reason.to_owned())?; + if crate::config::project_root(&self.location) != self.root { + return Err("rule project root changed during the operation".into()); + } + Ok(()) + } +} +impl std::fmt::Debug for RootBinding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RootBinding") + .field("root", &self.root) + .finish_non_exhaustive() + } +} + pub struct Definition { pub path: PathBuf, pub digest: String, @@ -274,6 +322,79 @@ pub fn trust_subset(location: &Path) -> TrustInventory { #[cfg(test)] mod tests { use super::*; + + #[test] + fn root_binding_tracks_native_ancestry_and_project_boundaries() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("checkout"); + let location = root.join("web/src"); + std::fs::create_dir_all(&location).unwrap(); + std::fs::create_dir(root.join(".git")).unwrap(); + let binding = RootBinding::capture(&location).unwrap(); + assert_eq!(binding.root(), root.canonicalize().unwrap()); + std::fs::write(location.join("app.ts"), "ordinary edit").unwrap(); + assert!(binding.verify().is_ok()); + // A nested checkout changes scope even though the physical directories remain. + std::fs::create_dir(root.join("web/.git")).unwrap(); + assert!(binding.verify().is_err()); + std::fs::remove_dir(root.join("web/.git")).unwrap(); + assert!(binding.verify().is_ok()); + std::fs::rename(root.join("web"), root.join("old-web")).unwrap(); + std::fs::create_dir_all(&location).unwrap(); + assert!( + binding.verify().is_err(), + "same path is a different ancestor" + ); + assert!(RootBinding::capture(&location).unwrap().verify().is_ok()); + } + + #[test] + fn root_binding_rejects_replaced_and_removed_checkouts() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("checkout"); + std::fs::create_dir(&root).unwrap(); + let binding = RootBinding::capture(&root).unwrap(); + std::fs::rename(&root, temp.path().join("retained")).unwrap(); + assert!(binding.verify().is_err()); + std::fs::create_dir(&root).unwrap(); + assert!( + binding.verify().is_err(), + "replacement must not borrow held identity" + ); + assert!(RootBinding::capture(&root).unwrap().verify().is_ok()); + } + + #[cfg(unix)] + #[test] + fn root_binding_accepts_a_declared_alias_but_refuses_later_link_substitution() { + use std::os::unix::fs::symlink; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("checkout"); + std::fs::create_dir(&root).unwrap(); + let alias = temp.path().join("alias"); + symlink(&root, &alias).unwrap(); + let binding = RootBinding::capture(&alias).unwrap(); + assert!(binding.verify().is_ok()); + let other = temp.path().join("other"); + std::fs::create_dir(&other).unwrap(); + std::fs::remove_file(&alias).unwrap(); + symlink(&other, &alias).unwrap(); + assert!( + binding.verify().is_err(), + "declared alias retargeted to another root" + ); + std::fs::remove_file(&alias).unwrap(); + symlink(&root, &alias).unwrap(); + assert!(binding.verify().is_ok()); + let retained = temp.path().join("retained"); + std::fs::rename(&root, &retained).unwrap(); + symlink(&retained, &root).unwrap(); + assert!( + binding.verify().is_err(), + "native lookup must not follow a substituted link" + ); + } + fn root() -> (tempfile::TempDir, PathBuf) { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 6dadc68a0..f1f0333df 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -2085,8 +2085,9 @@ impl Inner { let location = std::path::PathBuf::from(&invocation.directory); Some(ReadTarget { file: super::instructions::read_target(&location, settings.home.as_deref(), path), - location, + location: location.clone(), scope, + root: Arc::new(cyber_core::rules::RootBinding::capture(&location).ok()?), }) } @@ -2108,7 +2109,7 @@ impl Inner { let state = handle.state.lock().await; let targets: Vec<_> = targets .into_iter() - .map(|target| target.filter(|read| read.scope.is_current(&state))) + .map(|target| target.filter(|read| read.is_current(&state))) .collect(); let attached = Reminded::of(&state).instructions; drop(state); @@ -2120,16 +2121,9 @@ impl Inner { let nested = target.map_or_else(Default::default, |read| { // Resolve again after the effect: pre-call permission callbacks and // trust inventories may have been invalidated while the tool ran. - let Ok(settings) = inner.instruction_settings(&read.scope.directory) else { - return Default::default(); - }; - super::instructions::load_nested( - &global, - &read.location, - &read.file, - &settings, - &attached, - ) + read.load(&global, &attached, || { + inner.instruction_settings(&read.scope.directory) + }) }); loaded.push(nested); } @@ -2365,7 +2359,7 @@ impl ToolRun { if self .read .as_ref() - .is_some_and(|read| read.scope.is_current(state)) + .is_some_and(|read| read.is_current(state)) { nested } else { @@ -2420,6 +2414,41 @@ pub(super) struct ReadTarget { pub location: std::path::PathBuf, pub file: std::path::PathBuf, pub scope: ReminderScope, + pub root: Arc, +} +impl ReadTarget { + fn is_current(&self, state: &SessionState) -> bool { + self.scope.is_current(state) && self.root.verify().is_ok() + } + + fn load( + &self, + global: &std::path::Path, + attached: &std::collections::BTreeSet, + resolve: impl FnOnce() -> Result, + ) -> super::instructions::Instructions { + if self.root.verify().is_err() { + return Default::default(); + } + let Ok(settings) = resolve() else { + return Default::default(); + }; + if self.root.verify().is_err() { + return Default::default(); + } + let loaded = super::instructions::load_nested( + global, + &self.location, + &self.file, + &settings, + attached, + ); + if self.root.verify().is_ok() { + loaded + } else { + Default::default() + } + } } /// `InstructionsLoaded` `load_reason` for files a read attached. @@ -2804,6 +2833,8 @@ mod skill_reminder_tests { #[test] fn reminders_cannot_cross_an_epoch_or_location_change_during_loading() { + let root_dir = tempfile::tempdir().unwrap(); + let root = Arc::new(cyber_core::rules::RootBinding::capture(root_dir.path()).unwrap()); let state = || { let info = serde_json::from_value(serde_json::json!({ "id":"ses_rules", "title":"rules", "directory":"/repo", "parent_id":null, @@ -2830,6 +2861,7 @@ mod skill_reminder_tests { location: "/repo".into(), file: "/repo/web/app.ts".into(), scope, + root: root.clone(), }), }; match changed { @@ -2855,6 +2887,7 @@ mod skill_reminder_tests { location: state.info.directory.clone().into(), file: "/repo/web/app.ts".into(), scope, + root: root.clone(), }), ..run }; @@ -3065,6 +3098,7 @@ mod skill_reminder_tests { /// ones included, before the outcome was checked. #[test] fn only_successful_reads_load_nested_instructions() { + let root_dir = tempfile::tempdir().unwrap(); let read = ReadTarget { location: "/repo".into(), file: "/repo/web/app.ts".into(), @@ -3072,6 +3106,7 @@ mod skill_reminder_tests { directory: "/repo".into(), epoch: 1, }, + root: Arc::new(cyber_core::rules::RootBinding::capture(root_dir.path()).unwrap()), }; let run = |outcome| ToolRun { call_id: "call".into(), @@ -3104,6 +3139,7 @@ mod skill_reminder_tests { /// and the source's own error and structured value carry over. #[test] fn external_results_settle_framed_without_reminders() { + let root_dir = tempfile::tempdir().unwrap(); let (_temp, nested) = nested(&[("CYBER.md", "rule".into())]); let framed = "\nx\n"; @@ -3148,6 +3184,7 @@ mod skill_reminder_tests { directory: "/repo".into(), epoch: 1, }, + root: Arc::new(cyber_core::rules::RootBinding::capture(root_dir.path()).unwrap()), }), }; assert!(run.nested_target().is_none()); diff --git a/crates/cyber-server/tests/context_compaction.rs b/crates/cyber-server/tests/context_compaction.rs index de9ac5a6f..d0e838fde 100644 --- a/crates/cyber-server/tests/context_compaction.rs +++ b/crates/cyber-server/tests/context_compaction.rs @@ -732,3 +732,68 @@ async fn nested_instructions_resolve_settings_after_the_successful_operation() { ); assert_eq!(state.epoch.unwrap().reminded_instructions.len(), 1); } + +/// Same-path replacement must not load the new checkout's instruction files for +/// an operation captured against the old physical Location. +#[tokio::test] +async fn a_replaced_location_cannot_supply_instructions_for_an_inflight_read() { + let h = Harness::new(Setup { + scripts: vec![( + "test/main", + vec![ + tools(&[("read_old", "read", r#"{"path":"web/app.ts"}"#)]), + text("done"), + tools(&[("read_new", "read", r#"{"path":"web/app.ts"}"#)]), + text("done"), + ], + )], + ..Setup::default() + }); + std::fs::create_dir(h.repo.join("web")).unwrap(); + std::fs::write(h.repo.join("web/app.ts"), "original file").unwrap(); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::Gated); + let id = h.session().await; + h.runtime + .admit(&id, admit("read old checkout")) + .await + .unwrap(); + tokio::time::timeout( + std::time::Duration::from_secs(5), + h.tools.started.notified(), + ) + .await + .unwrap(); + std::fs::rename(&h.repo, h.dir.path().join("old-checkout")).unwrap(); + std::fs::create_dir_all(h.repo.join("web")).unwrap(); + std::fs::create_dir(h.repo.join(".git")).unwrap(); + std::fs::write(h.repo.join("web/app.ts"), "replacement file").unwrap(); + std::fs::write(h.repo.join("web/CYBER.md"), "replacement checkout guidance").unwrap(); + h.tools.release.notify_one(); + h.settle(&id).await; + let state = h.state(&id).await; + assert_eq!(state.calls["read_old"].output.as_deref(), Some("done")); + assert!(state.epoch.unwrap().reminded_instructions.is_empty()); + h.tools + .behavior + .lock() + .unwrap() + .insert("read".into(), Behavior::Return("replacement file".into())); + h.runtime + .admit(&id, admit("read new checkout")) + .await + .unwrap(); + h.settle(&id).await; + let state = h.state(&id).await; + assert!( + state.calls["read_new"] + .output + .as_deref() + .unwrap() + .contains("replacement checkout guidance") + ); + assert_eq!(state.epoch.unwrap().reminded_instructions.len(), 1); +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 1da0bab56..e7a6daf02 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Nested read delivery now reloads settings after success and checks captured Location/Epoch before discovery and under the settlement lock. Finish edit/patch/partial-effect delivery, native operation/config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Nested read delivery now reloads settings after success and checks captured Location/Epoch plus native Location ancestry and project boundary before/after discovery and under the settlement lock. Finish edit/patch/partial-effect delivery, native operation/config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index d32aa829b..1b33a02fe 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1221,7 +1221,7 @@ Project rule bytes are included in the existing checkout trust digest. Review wi Definition reads use the fenced native reader: linked files/directories, hard links and special files are refused. The reader caps a definition at 1 MiB; frontmatter is limited to 16 KiB, with up to 64 relative patterns of 256 bytes each. A project inventory covers up to 128 definitions and 4 MiB, with bounded directory enumeration. These review/discovery bounds are separate from the smaller rendered instruction budgets. Unknown inventories grant no rule authority. `cyber doctor` reports held, refused and truncated baseline rules. -The current local implementation activates scoped rules after successful built-in reads and records typed root/path/digest/approval bindings separately from legacy instruction paths. Nested discovery resolves settings after the successful read and checks that its captured Location and Context Epoch still apply before loading and before settlement. A suppressed reminder remains eligible for a later operation. Edit/patch and partial-settlement delivery, native operation and exact configuration-generation binding, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. +The current local implementation activates scoped rules after successful built-in reads and records typed root/path/digest/approval bindings separately from legacy instruction paths. Nested discovery resolves settings after the successful read and checks that its captured Location and Context Epoch still apply before loading and before settlement. The read retains the physical Location's native directory identities and withholds reminders after root/ancestor replacement, removal, link substitution or a changed project boundary. A suppressed reminder remains eligible for a later operation. These checks do not yet bind the actual file handle used by the operation. Edit/patch and partial-settlement delivery, native operation and exact configuration-generation binding, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 741fa265d..4a0695b35 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -26,7 +26,7 @@ Rule definitions use the existing fenced native source reader (no linked/hard-li Definition discovery bounds are 128 project files, 4096 examined entries per rules directory, 1 MiB per fenced definition and 4 MiB per inventory/loader; frontmatter is capped at 16 KiB and 64 portable relative globs of at most 256 bytes. Invalid/null/empty `paths` and unknown authority-bearing keys are errors, never baseline fallbacks. Rule identity includes root, definition path, exact bytes and approval generation. Successful delivery stores optional typed `rule_reminders` separately from the legacy canonical `instruction_reminders`; historical events default to no rule bindings. Exact native operation/config-generation/Epoch binding and withdrawal/restart behavior remain required under 4.2–4.4. -Nested read discovery resolves instruction settings after a successful effect. Its captured Session Location and Context Epoch must still be active before discovery and again under the settlement lock. An absent/stale/replaced Epoch or changed Location suppresses the reminder without recording delivery; normal group settlement and exact `/approve` replay share this check. This guards the runtime context boundary, but does not replace native operation receipts, exact configuration-generation fencing through commit, or revocation withdrawal. +Nested read discovery resolves instruction settings after a successful effect. Its captured Session Location and Context Epoch must still be active before discovery and again under the settlement lock. An absent/stale/replaced Epoch or changed Location suppresses the reminder without recording delivery; normal group settlement and exact `/approve` replay share this check. The captured read also retains the native directory identities from the volume root through the physical Location, verifies the project-root boundary, and rechecks those bindings before/after discovery and under settlement. Unix retains directory handles to prevent inode reuse; Windows releases handles after capture and verifies full native IDs, so the observation does not freeze editor/worktree renames. Ordinary file content edits do not invalidate the root. Replacement/removal, changed ancestors, linked substitution, a retargeted declared Location alias or a changed project boundary withhold the reminder. These point-in-time context/root checks do not prove which file handle performed the effect: actual per-file operation receipts (including partial writes), exact configuration-generation fencing through commit and revocation withdrawal remain required. Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index 92e153614..cfb5b8160 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -61,3 +61,10 @@ - **THEN** the read result SHALL settle without delivering that operation's nested reminder into the new context - **AND** the suppressed reminder SHALL NOT consume once-per-Epoch delivery - **AND** nested discovery SHALL resolve current instruction settings after the successful operation instead of retaining pre-effect permission callbacks + +#### Scenario: Physical Location replacement during a read +- **GIVEN** a file read started with a captured physical Location and project root +- **WHEN** that Location or any native ancestor is replaced, removed or substituted with a link before nested discovery or settlement +- **THEN** same-path replacement sources SHALL NOT supply reminders for the old operation +- **AND** the operation result SHALL remain intact and its suppressed reminder SHALL NOT consume delivery in the active Epoch +- **AND** a later operation captured against the replacement Location SHALL remain independently eligible diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 4c31c824c..c96a05a78 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1706,3 +1706,49 @@ findings within these changes: existing runtime/host boundaries and atomic settlement are retained, unavailable configuration holds reminders, both writer paths revalidate the context, and regressions cover suppression and retry. Whole-goal review and source-bound native acceptance remain open. + + +Native Location observation continuation (2026-11-10): add a metadata-only +DirectorySnapshot using the existing no-follow native source reader's directory +chain and identities. RootBinding retains the physical Location and project root, +checks the declared Location alias still names that directory, and revalidates +all native ancestors and the nearest project boundary. Unix retains handles; +Windows retains full native IDs without holding directories open through the +operation, preserving rename support. Runtime read targets retain the binding; +normal and exact-override settlement check it before discovery/settings, after +settings/discovery and under the settlement lock. No new file-operation or source +permission is granted. Root/ancestor replacement, removal, symlink substitution, +alias retargeting and changed nested checkout boundaries withhold reminders. +Ordinary edits preserve the binding. A gated runtime read regression verifies a +replacement checkout cannot supply nested guidance for the old call, while a +fresh operation can load it without the old call consuming delivery. The existing +rule/override and budget behavior remains covered by production App regressions. +Checkpoint review found and fixed the declared-alias retargeting case; its native +symlink regression now covers both alias retargeting and physical-path link +substitution. These are point-in-time native observations, not proof of the file +handle used by the effect. Durable rule identities still use logical root paths; +actual per-file/native-operation receipts, root incarnation in durable provenance, +edit/patch/partial-success delivery, exact configuration-generation fencing, +revocation withdrawal, restart and inspection remain open. No parent task closes. + +Next producer integration points from code inspection: the successful read in +`cyber-tools/src/tools/fs.rs`, `guarded_write` after its write, and each completed +`write_one` in `tools/patch.rs`. Patch completion paths already distinguish files +applied before failure. Collect actual authorized per-file effects there, including +hook-rewritten inputs, rather than deriving delivery from pre-hook arguments or +the overall tool status. RootBinding alone must not be treated as that receipt. + +Final local verification after the alias fix: eight filtered Core tests passed +(seven rules tests plus the existing untrusted-rule table test), server library +455 passed / one ignored subprocess lock probe, context/compaction 22 passed, +production App instruction integration 15 passed, and Core/Server/App all-target +Clippy with `-D warnings` passed. Formatting/diff checks, workflow YAML parsing, +generated SDK consistency and CI-pinned strict OpenSpec (100 items) passed; +spec lint reports zero errors and 25 warnings. Native C/linker warnings were +observed without assigning their origin. Main was fetched and remains fully +integrated at `8266db6c`. SDK runtime, entire workspace and Windows/native platform +acceptance were not rerun locally for this increment. Workflow source filters +include the shared native snapshot helper; its existing rules/context/App steps +exercise the new cases. Current-source native execution and whole-goal review +remain required before delivery; no goal/task acceptance is inferred from these +local checks. From 83af7dcab8d7b8683025f8c6a0ade421590e12c0 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 18:31:04 -0400 Subject: [PATCH 79/87] Bind scoped rule delivery to observed file reads --- .github/workflows/non-git-snapshot-proof.yml | 5 + crates/cyber-app/src/host.rs | 16 ++ crates/cyber-app/tests/instructions.rs | 76 +++++++ crates/cyber-core/src/import/mod.rs | 4 +- crates/cyber-core/src/import/snapshot.rs | 123 +++++++++- crates/cyber-core/src/memory/windows.rs | 14 ++ crates/cyber-core/src/rules.rs | 162 +++++++++++++ .../cyber-server/src/runtime/auto_override.rs | 2 + crates/cyber-server/src/runtime/context.rs | 2 + crates/cyber-server/src/runtime/drain.rs | 93 +++++++- crates/cyber-server/src/runtime/events.rs | 32 +++ crates/cyber-server/src/runtime/host.rs | 23 ++ .../cyber-server/src/runtime/instructions.rs | 21 +- crates/cyber-server/src/runtime/mod.rs | 9 +- crates/cyber-server/src/runtime/model.rs | 8 + crates/cyber-tools/src/host.rs | 214 +++++++++++------- crates/cyber-tools/src/host/attachments.rs | 8 +- crates/cyber-tools/src/host/command_skills.rs | 1 + crates/cyber-tools/src/host/files.rs | 1 + .../cyber-tools/src/host/template_expand.rs | 1 + .../cyber-tools/src/host/workflow_agents.rs | 2 + .../src/host/workflow_integration.rs | 3 + crates/cyber-tools/src/host/workflow_solve.rs | 1 + crates/cyber-tools/src/mcp/sampler.rs | 1 + crates/cyber-tools/src/registered.rs | 1 + crates/cyber-tools/src/sandboxing.rs | 1 + crates/cyber-tools/src/tools/agent.rs | 1 + crates/cyber-tools/src/tools/binary/patch.rs | 1 + crates/cyber-tools/src/tools/fs.rs | 57 ++++- crates/cyber-tools/src/worktrees.rs | 6 + crates/cyber-tools/src/worktrees/child.rs | 2 + crates/cyber-tools/src/worktrees/recovery.rs | 2 + docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 4 +- .../complete-daily-coding-surfaces/design.md | 2 + .../specs/system-context/spec.md | 8 + .../complete-daily-coding-surfaces/tasks.md | 45 ++++ 37 files changed, 846 insertions(+), 108 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 724f603b8..f7c18365a 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -29,10 +29,14 @@ on: - crates/cyber-core/src/config/** - crates/cyber-core/src/rules.rs - crates/cyber-core/src/import/snapshot.rs + - crates/cyber-core/src/memory/windows.rs + - crates/cyber-core/tests/import_snapshot.rs - crates/cyber-core/src/import/mod.rs - crates/cyber-core/src/lib.rs - crates/cyber-core/tests/config.rs - crates/cyber-tools/src/host.rs + - crates/cyber-tools/src/tools/fs.rs + - crates/cyber-server/src/runtime/auto_override.rs - crates/cyber-tools/tests/instruction_config.rs - crates/cyber-tools/tests/path_skills.rs - crates/cyber-server/src/runtime/context.rs @@ -200,6 +204,7 @@ jobs: mkdir -p "$TMPDIR" cargo test --locked -p cyber-core --test config instruction_limits -- --nocapture 2>&1 | tee instruction-limits-proof.log cargo test --locked -p cyber-core --lib rules:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-core --test import_snapshot -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-core --test config rule_definition_bytes -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::context:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::instructions:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log diff --git a/crates/cyber-app/src/host.rs b/crates/cyber-app/src/host.rs index 611ca20d3..9fac4765b 100644 --- a/crates/cyber-app/src/host.rs +++ b/crates/cyber-app/src/host.rs @@ -365,6 +365,22 @@ impl ToolHost for AppHost { self.builtin.execute(call, cancel) } + fn execute_observed( + &self, + call: Invocation, + cancel: CancellationToken, + ) -> BoxFuture<'_, cyber_server::runtime::ObservedExecution> { + if self.remote.has(&call.name) { + return Box::pin(async move { + cyber_server::runtime::ObservedExecution { + outcome: self.execute(call, cancel).await, + reads: None, + } + }); + } + self.builtin.execute_observed(call, cancel) + } + fn lifecycle_hooks( &self, event: cyber_server::runtime::LifecycleHook, diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index c3ad4fc62..b0f7cfffa 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -190,6 +190,38 @@ async fn project_rules_require_review_then_separate_baseline_from_matching_reads .unwrap() .starts_with("sha256:") ); + let proof = &settled.data["rule_read"]; + assert_eq!( + proof["evidence"]["path"], + f.repo.join("web/src/app.ts").to_str().unwrap() + ); + assert_eq!(proof["evidence"]["root_identity"], binding["native_root"]); + assert!( + proof["evidence"]["file_identity"] + .as_str() + .unwrap() + .starts_with("sha256:") + ); + assert_eq!( + proof["operation_key"], + format!("{session}:{}", settled.data["call_id"].as_str().unwrap()) + ); + assert_eq!(proof["attempt"], 1); + assert_eq!(proof["location"], f.repo.to_str().unwrap()); + for field in ["epoch", "attempt", "operation_key", "location"] { + let mut changed = events.clone(); + let event = changed + .iter_mut() + .find(|event| event.id == settled.id) + .unwrap(); + event.data["rule_read"][field] = if matches!(field, "operation_key" | "location") { + json!("other-call") + } else { + json!(999) + }; + let replayed = cyber_server::runtime::SessionState::replay(&changed).unwrap(); + assert!(replayed.epoch.unwrap().reminded_rules.is_empty(), "{field}"); + } let replayed = cyber_server::runtime::SessionState::replay(&events).unwrap(); assert_eq!(replayed.epoch.unwrap().reminded_rules, ids); f.prompt(&session, &f.repo, "read the app again", 2).await; @@ -820,3 +852,47 @@ async fn instruction_files_never_import_a_file_a_read_deny_rule_covers() { assert_secret_stays_out(&f, "config/secrets.json").await; f.app.runtime.shutdown().await; } + +/// The provider's input is not operation evidence: PreToolUse can rewrite it. +#[cfg(unix)] +#[tokio::test] +async fn rewritten_reads_deliver_only_rules_for_the_file_actually_opened() { + let rewrite = json!({"updated_input":{"path":"db/schema.sql"}}); + let command = format!("cat >/dev/null; printf '%s' '{rewrite}'"); + let f = Fixture::start_with(json!({"hooks":{"PreToolUse":[{"matcher":"read","hooks":[{ + "type":"command","id":"redirect-read","command":command + }]}]}})) + .await; + f.write("web/src/app.ts", "ORIGINAL_WEB_BYTES"); + f.write("db/schema.sql", "ACTUAL_DATABASE_BYTES"); + f.write( + ".cyber/rules/web.md", + "---\npaths: ['web/**']\n---\nWRONG_WEB_RULE", + ); + f.write( + ".cyber/rules/db.md", + "---\npaths: ['db/**']\n---\nACTUAL_DB_RULE", + ); + f.approve(&f.repo); + let id = f.session(&f.repo).await; + f.prompt(&id, &f.repo, "read the app first", 2).await; + let result = last_tool_result(&f.last_request()); + assert!(result.contains("ACTUAL_DATABASE_BYTES"), "{result}"); + assert!(result.contains("ACTUAL_DB_RULE"), "{result}"); + assert!(!result.contains("ORIGINAL_WEB_BYTES"), "{result}"); + assert!(!result.contains("WRONG_WEB_RULE"), "{result}"); + let events = f.app.store.read_events(&id, -1, 500).unwrap().events; + let settled = events + .iter() + .find(|event| event.data["rule_read"].is_object()) + .unwrap(); + assert_eq!( + settled.data["rule_read"]["evidence"]["path"], + f.repo.join("db/schema.sql").to_str().unwrap() + ); + let state = f.app.runtime.state(&id).await.unwrap(); + let call = &state.calls[settled.data["call_id"].as_str().unwrap()]; + assert_eq!(call.input.as_ref().unwrap()["path"], "web/src/app.ts"); + assert_eq!(state.epoch.unwrap().reminded_rules.len(), 1); + f.app.runtime.shutdown().await; +} diff --git a/crates/cyber-core/src/import/mod.rs b/crates/cyber-core/src/import/mod.rs index c1c49250b..ce1bb6d84 100644 --- a/crates/cyber-core/src/import/mod.rs +++ b/crates/cyber-core/src/import/mod.rs @@ -45,7 +45,9 @@ pub use preview::{ }; pub use provenance::SourceReference; pub use snapshot::SourceSnapshot; -pub(crate) use snapshot::{DirectorySnapshot, command_directory, verify_command_directory}; +pub(crate) use snapshot::{ + DirectorySnapshot, FileRead, FileReadReceipt, command_directory, verify_command_directory, +}; mod opencode_providers; pub use opencode_providers::{OpenCodeProviderConfig, opencode_provider_config}; diff --git a/crates/cyber-core/src/import/snapshot.rs b/crates/cyber-core/src/import/snapshot.rs index 687ec5801..68244ad12 100644 --- a/crates/cyber-core/src/import/snapshot.rs +++ b/crates/cyber-core/src/import/snapshot.rs @@ -29,22 +29,33 @@ enum Identity { Windows(crate::memory::windows::FileIdentity), } fn identity(file: &File, path: &Path) -> Result { + observed_identity(file, path, false) +} +fn observed_identity( + file: &File, + path: &Path, + allow_file_links: bool, +) -> Result { #[cfg(unix)] { use std::os::unix::fs::MetadataExt; let m = file .metadata() .map_err(|_| error(path, "source identity is unavailable"))?; - if m.is_file() && m.nlink() != 1 { + if !allow_file_links && m.is_file() && m.nlink() != 1 { return Err(error(path, "hard-linked source file is unsupported")); } Ok(Identity::Unix(m.dev(), m.ino())) } #[cfg(windows)] { - crate::memory::windows::identity(file) - .map(Identity::Windows) - .map_err(|_| error(path, "source native identity or reparse check failed")) + (if allow_file_links { + crate::memory::windows::observation_identity(file) + } else { + crate::memory::windows::identity(file) + }) + .map(Identity::Windows) + .map_err(|_| error(path, "source native identity or reparse check failed")) } #[cfg(not(any(unix, windows)))] { @@ -73,6 +84,13 @@ struct FileState { modified: SystemTime, } fn state(file: &File, path: &Path) -> Result { + observed_state(file, path, false) +} +fn observed_state( + file: &File, + path: &Path, + allow_file_links: bool, +) -> Result { let metadata = file .metadata() .map_err(|_| error(path, "source file metadata is unavailable"))?; @@ -80,7 +98,7 @@ fn state(file: &File, path: &Path) -> Result { return Err(error(path, "source is not a regular file")); } Ok(FileState { - identity: identity(file, path)?, + identity: observed_identity(file, path, allow_file_links)?, len: metadata.len(), modified: metadata .modified() @@ -152,6 +170,96 @@ impl DirectorySnapshot { } } +fn fingerprint(identities: &[Identity]) -> String { + let mut hash = Sha256::new(); + hash.update(b"cyber-native-observation-v1\0"); + for identity in identities { + match identity { + #[cfg(unix)] + Identity::Unix(device, inode) => { + hash.update([1]); + hash.update(device.to_le_bytes()); + hash.update(inode.to_le_bytes()); + } + #[cfg(windows)] + Identity::Windows(id) => { + hash.update([2]); + hash.update(id.volume.to_le_bytes()); + hash.update(id.file); + } + } + } + format!("sha256:{:x}", hash.finalize()) +} +impl DirectorySnapshot { + pub(crate) fn ancestor_identity(&self, ancestor: &Path) -> Option { + let depth = self.path.strip_prefix(ancestor).ok()?.components().count(); + Some(fingerprint( + &self.bindings[..self.bindings.len().checked_sub(depth)?], + )) + } +} + +/// An actual read performed through the captured no-follow file handle. Unlike +/// definition snapshots this preserves the read tool's existing file-size contract. +pub(crate) struct FileRead { + path: PathBuf, + opened: OpenSource, +} +pub(crate) struct FileReadReceipt { + path: PathBuf, + bindings: Vec, + state: FileState, + _retained: Option, +} +impl FileRead { + pub(crate) fn open(path: &Path) -> Result { + Ok(Self { + path: path.into(), + opened: OpenSource::open_observed(path, true)?, + }) + } + pub(crate) fn read(mut self) -> std::io::Result<(Vec, FileReadReceipt)> { + let mut bytes = Vec::new(); + self.opened.file.read_to_end(&mut bytes)?; + let bindings = self.opened.bindings.clone(); + let state = self.opened.state.clone(); + #[cfg(windows)] + let retained = { + drop(self.opened); + None + }; + #[cfg(not(windows))] + let retained = Some(self.opened); + Ok(( + bytes, + FileReadReceipt { + path: self.path, + bindings, + state, + _retained: retained, + }, + )) + } +} +impl FileReadReceipt { + pub(crate) fn verify(&self) -> Result<(), DiscoveryError> { + let current = OpenSource::open_observed(&self.path, true)?; + if current.bindings != self.bindings || current.state.identity != self.state.identity { + return Err(error( + &self.path, + "read target changed since the observed operation", + )); + } + Ok(()) + } + pub(crate) fn identity(&self) -> String { + let mut identities = self.bindings.clone(); + identities.push(self.state.identity.clone()); + fingerprint(&identities) + } +} + /// Internal read-time command discovery keeps every directory binding live during enumeration. pub(crate) fn command_directory(path: &Path) -> Result, DiscoveryError> { directory_chain(path).map(|(directories, _)| directories) @@ -163,6 +271,9 @@ pub(crate) fn verify_command_directory(dir: &Dir, path: &Path) -> Result<(), Dis impl OpenSource { fn open(path: &Path) -> Result { + Self::open_observed(path, false) + } + fn open_observed(path: &Path, allow_file_links: bool) -> Result { let name = path .file_name() .ok_or_else(|| error(path, "source file name is missing"))?; @@ -188,7 +299,7 @@ impl OpenSource { .open_with(name, &options) .map_err(|_| error(path, "source file cannot be opened without links"))? .into_std(); - let state = state(&file, path)?; + let state = observed_state(&file, path, allow_file_links)?; Ok(Self { _directories: directories, bindings, diff --git a/crates/cyber-core/src/memory/windows.rs b/crates/cyber-core/src/memory/windows.rs index 636189a56..0d73edea9 100644 --- a/crates/cyber-core/src/memory/windows.rs +++ b/crates/cyber-core/src/memory/windows.rs @@ -76,6 +76,20 @@ pub fn identity(file: &File) -> Result { if info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY == 0 { verify_regular(file)?; } + full_file_id(file) +} + +/// Metadata-only identity of an operation target. This grants no memory-storage +/// authority: memory callers retain `identity`/`verify_regular` and their link checks. +pub(crate) fn observation_identity(file: &File) -> Result { + information(file)?; // all reparse types remain refused + if !file.metadata()?.is_file() { + return Err(refusal("expected a regular observation target")); + } + full_file_id(file) +} + +fn full_file_id(file: &File) -> Result { let mut id = FILE_ID_INFO::default(); if unsafe { GetFileInformationByHandleEx( diff --git a/crates/cyber-core/src/rules.rs b/crates/cyber-core/src/rules.rs index d835b89fd..c8224bb63 100644 --- a/crates/cyber-core/src/rules.rs +++ b/crates/cyber-core/src/rules.rs @@ -59,6 +59,99 @@ impl std::fmt::Debug for RootBinding { } } +/// Metadata describing a successful read; these hashes are observations, never +/// authority to perform another operation or reopen a source after restart. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ReadEvidence { + pub root: PathBuf, + pub path: PathBuf, + pub root_identity: String, + pub file_identity: String, +} +pub struct ObservedRead { + declared: PathBuf, + physical: PathBuf, + root: RootBinding, + reader: crate::import::FileRead, +} +pub struct ReadReceipt { + declared: PathBuf, + root: RootBinding, + native: crate::import::FileReadReceipt, + evidence: ReadEvidence, +} +impl RootBinding { + pub fn identity(&self) -> String { + self.native + .ancestor_identity(&self.root) + .expect("project root is an ancestor") + } +} +impl ObservedRead { + pub fn open(location: &Path, path: &Path, authorized: &Path) -> Result { + let root = RootBinding::capture(location)?; + let declared = std::path::absolute(path).map_err(|_| "read path is invalid")?; + let physical = path + .canonicalize() + .map_err(|_| "read target is unavailable")?; + if physical != authorized { + return Err("read path changed after authorization".into()); + } + if !physical.starts_with(root.root()) { + return Err("read is outside the captured root".into()); + } + // Receipts are persisted as exact paths; do not mint lossy identities. + if physical.to_str().is_none() || root.root().to_str().is_none() { + return Err("read receipt paths must be UTF-8".into()); + } + let reader = crate::import::FileRead::open(&physical).map_err(|e| e.reason.to_owned())?; + root.verify()?; + Ok(Self { + declared, + physical, + root, + reader, + }) + } + pub fn read(self) -> std::io::Result<(Vec, ReadReceipt)> { + let (bytes, native) = self.reader.read()?; + let evidence = ReadEvidence { + root: self.root.root().into(), + path: self.physical, + root_identity: self.root.identity(), + file_identity: native.identity(), + }; + Ok(( + bytes, + ReadReceipt { + declared: self.declared, + root: self.root, + native, + evidence, + }, + )) + } +} +impl ReadReceipt { + pub fn evidence(&self) -> &ReadEvidence { + &self.evidence + } + pub fn verify(&self) -> Result<(), String> { + self.root.verify()?; + if self.declared.canonicalize().ok().as_ref() != Some(&self.evidence.path) { + return Err("declared read target changed since the operation".into()); + } + self.native.verify().map_err(|e| e.reason.to_owned()) + } +} +impl std::fmt::Debug for ReadReceipt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReadReceipt") + .field("evidence", &self.evidence) + .finish_non_exhaustive() + } +} + pub struct Definition { pub path: PathBuf, pub digest: String, @@ -332,6 +425,10 @@ mod tests { std::fs::create_dir(root.join(".git")).unwrap(); let binding = RootBinding::capture(&location).unwrap(); assert_eq!(binding.root(), root.canonicalize().unwrap()); + assert_eq!( + binding.identity(), + RootBinding::capture(&root).unwrap().identity() + ); std::fs::write(location.join("app.ts"), "ordinary edit").unwrap(); assert!(binding.verify().is_ok()); // A nested checkout changes scope even though the physical directories remain. @@ -395,6 +492,71 @@ mod tests { ); } + #[test] + fn actual_reads_retain_file_and_root_identity_without_the_definition_size_cap() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("repo"); + std::fs::create_dir(&root).unwrap(); + let path = root.join("large.txt"); + let content = vec![b'x'; 1024 * 1024 + 1]; + std::fs::write(&path, &content).unwrap(); + let reader = ObservedRead::open(&root, &path, &path.canonicalize().unwrap()).unwrap(); + let (bytes, receipt) = reader.read().unwrap(); + assert_eq!(bytes, content); + assert!(receipt.verify().is_ok()); + assert_eq!(receipt.evidence().path, path.canonicalize().unwrap()); + assert_eq!( + receipt.evidence().root_identity, + RootBinding::capture(&root).unwrap().identity() + ); + // Ordinary writes to the same inode do not erase the earlier read operation. + std::fs::write(&path, "later content").unwrap(); + assert!(receipt.verify().is_ok()); + std::fs::rename(&path, root.join("retained.txt")).unwrap(); + std::fs::write(&path, &content).unwrap(); + assert!( + receipt.verify().is_err(), + "a new inode cannot borrow the actual read" + ); + } + + #[test] + fn workspace_hard_links_are_observable_but_definition_hard_links_are_refused() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("file.md"); + let link = temp.path().join("link.md"); + std::fs::write(&path, "workspace content").unwrap(); + std::fs::hard_link(&path, &link).unwrap(); + let (bytes, receipt) = + ObservedRead::open(temp.path(), &link, &link.canonicalize().unwrap()) + .unwrap() + .read() + .unwrap(); + assert_eq!(bytes, b"workspace content"); + assert!(receipt.verify().is_ok()); + assert!(Definition::read(&link).is_err()); + } + + #[cfg(unix)] + #[test] + fn observed_read_uses_its_open_handle_and_refuses_retargeted_aliases() { + use std::os::unix::fs::symlink; + let temp = tempfile::tempdir().unwrap(); + let a = temp.path().join("a.txt"); + let b = temp.path().join("b.txt"); + let alias = temp.path().join("alias.txt"); + std::fs::write(&a, "actual original bytes").unwrap(); + std::fs::write(&b, "different bytes").unwrap(); + symlink(&a, &alias).unwrap(); + let reader = ObservedRead::open(temp.path(), &alias, &a.canonicalize().unwrap()).unwrap(); + std::fs::remove_file(&alias).unwrap(); + symlink(&b, &alias).unwrap(); + let (bytes, receipt) = reader.read().unwrap(); + assert_eq!(bytes, b"actual original bytes"); + assert!(receipt.verify().is_err()); + assert!(ObservedRead::open(temp.path(), &alias, &a.canonicalize().unwrap()).is_err()); + } + fn root() -> (tempfile::TempDir, PathBuf) { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); diff --git a/crates/cyber-server/src/runtime/auto_override.rs b/crates/cyber-server/src/runtime/auto_override.rs index 2601b99b1..4107b4dca 100644 --- a/crates/cyber-server/src/runtime/auto_override.rs +++ b/crates/cyber-server/src/runtime/auto_override.rs @@ -474,6 +474,7 @@ impl Inner { let nested = nested.pop().unwrap_or_default(); let mut state = handle.state.lock().await; let nested = run.current_nested(&state, nested); + let proof = run.read_proof(); let mut seen = super::drain::Reminded::of(&state); let (settled, loaded) = super::drain::settlement_with_reminders( &replay.id, @@ -486,6 +487,7 @@ impl Inner { .finalize_skill_output(&state.info.directory, output, reminder) }, ); + let settled = super::drain::with_read_proof(settled, proof); let session_id = state.info.id.clone(); let settled = self.with_call_metadata(&session_id, &replay.id, &state.info.directory, settled); diff --git a/crates/cyber-server/src/runtime/context.rs b/crates/cyber-server/src/runtime/context.rs index c55a5123e..dbd8929f2 100644 --- a/crates/cyber-server/src/runtime/context.rs +++ b/crates/cyber-server/src/runtime/context.rs @@ -262,6 +262,8 @@ mod tests { home: None, read_gate: None, rule_gate: None, + scoped_rules: true, + native_root: None, }), today: Some("2026-10-03".into()), } diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index f1f0333df..64b7ac92a 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -1820,6 +1820,7 @@ impl Inner { let mut loaded = Vec::new(); for (run, nested) in outcomes.into_iter().zip(nested) { let nested = run.current_nested(&state, nested); + let proof = run.read_proof(); let (settled, instructions) = settlement_with_reminders( &run.call_id, &run.def, @@ -1831,6 +1832,7 @@ impl Inner { .finalize_skill_output(&directory, output, reminder) }, ); + let settled = with_read_proof(settled, proof); events.push(self.with_call_metadata(&session_id, &run.call_id, &directory, settled)); loaded.extend(instructions); stops.extend(self.take_call_hook_effects(&session_id, &run.call_id, &mut events)); @@ -2050,7 +2052,7 @@ impl Inner { read, }; } - let run = self.tools.execute(invocation, cancel.clone()); + let run = self.tools.execute_observed(invocation, cancel.clone()); tokio::pin!(run); let outcome = tokio::select! { out = &mut run => out, @@ -2058,14 +2060,27 @@ impl Inner { Ok(outcome) => outcome, Err(_) => { handle.location_uncertain.store(true, std::sync::atomic::Ordering::SeqCst); - ToolOutcome::Aborted + super::host::ObservedExecution { outcome: ToolOutcome::Aborted, reads: Some(Vec::new()) } } }, }; + let read = match outcome.reads { + None => read, + Some(mut receipts) if receipts.len() == 1 => read.and_then(|mut target| { + let receipt = receipts.pop().unwrap(); + if target.root.identity() != receipt.evidence().root_identity { + return None; + } + target.file = receipt.evidence().path.clone(); + target.receipt = Some(receipt); + Some(target) + }), + Some(_) => None, + }; ToolRun { call_id, def, - outcome, + outcome: outcome.outcome, read, } } @@ -2088,6 +2103,9 @@ impl Inner { location: location.clone(), scope, root: Arc::new(cyber_core::rules::RootBinding::capture(&location).ok()?), + receipt: None, + operation_key: invocation.operation_key.clone(), + attempt: invocation.attempt, }) } @@ -2328,6 +2346,7 @@ fn settled( skill_reminders: Vec::new(), instruction_reminders: Vec::new(), rule_reminders: Vec::new(), + rule_read: None, structured_output: None, call_id: call_id.into(), status, @@ -2349,6 +2368,16 @@ pub(super) struct ToolRun { } impl ToolRun { + pub(super) fn read_proof(&self) -> Option { + let target = self.read.as_ref()?; + Some(super::events::RuleRead { + evidence: target.receipt.as_ref()?.evidence().clone(), + epoch: target.scope.epoch, + location: target.scope.directory.clone(), + operation_key: target.operation_key.clone(), + attempt: target.attempt, + }) + } /// Recheck under the settlement lock, after filesystem discovery. A Location /// switch or Epoch replacement during discovery must not consume new identities. pub(super) fn current_nested( @@ -2415,10 +2444,20 @@ pub(super) struct ReadTarget { pub file: std::path::PathBuf, pub scope: ReminderScope, pub root: Arc, + pub receipt: Option>, + pub operation_key: String, + pub attempt: u32, } impl ReadTarget { + fn valid_native(&self) -> bool { + self.root.verify().is_ok() + && self + .receipt + .as_ref() + .is_none_or(|receipt| receipt.verify().is_ok()) + } fn is_current(&self, state: &SessionState) -> bool { - self.scope.is_current(state) && self.root.verify().is_ok() + self.scope.is_current(state) && self.valid_native() } fn load( @@ -2427,15 +2466,20 @@ impl ReadTarget { attached: &std::collections::BTreeSet, resolve: impl FnOnce() -> Result, ) -> super::instructions::Instructions { - if self.root.verify().is_err() { + if !self.valid_native() { return Default::default(); } - let Ok(settings) = resolve() else { + let Ok(mut settings) = resolve() else { return Default::default(); }; - if self.root.verify().is_err() { + if !self.valid_native() { return Default::default(); } + settings.scoped_rules = self.receipt.is_some(); + settings.native_root = self + .receipt + .as_ref() + .map(|receipt| receipt.evidence().root_identity.clone()); let loaded = super::instructions::load_nested( global, &self.location, @@ -2443,7 +2487,7 @@ impl ReadTarget { &settings, attached, ); - if self.root.verify().is_ok() { + if self.valid_native() { loaded } else { Default::default() @@ -2451,6 +2495,23 @@ impl ReadTarget { } } +pub(super) fn with_read_proof( + mut settled: cyber_store::NewEvent, + proof: Option, +) -> cyber_store::NewEvent { + if settled + .data + .get("rule_reminders") + .and_then(serde_json::Value::as_array) + .is_some_and(|rules| !rules.is_empty()) + && let Some(proof) = proof + { + settled.data["rule_read"] = + serde_json::to_value(proof).expect("rule read metadata serializes"); + } + settled +} + /// `InstructionsLoaded` `load_reason` for files a read attached. pub(super) const NESTED_LOAD_REASON: &str = "nested_read"; @@ -2558,6 +2619,7 @@ pub(super) fn settlement_with_reminders( skill_reminders: names, instruction_reminders: attached, rule_reminders: rule_bindings, + rule_read: None, structured_output: value, call_id: call_id.into(), status: if failed { @@ -2657,6 +2719,7 @@ pub(super) fn settlement( skill_reminders: Vec::new(), instruction_reminders: Vec::new(), rule_reminders: Vec::new(), + rule_read: None, structured_output: None, call_id: call_id.into(), status: CallStatus::Ok, @@ -2690,6 +2753,7 @@ pub(super) fn settlement( skill_reminders: Vec::new(), instruction_reminders: Vec::new(), rule_reminders: Vec::new(), + rule_read: None, structured_output: Some(value), call_id: call_id.into(), status: CallStatus::Ok, @@ -2714,6 +2778,7 @@ pub(super) fn settlement( skill_reminders: Vec::new(), instruction_reminders: Vec::new(), rule_reminders: Vec::new(), + rule_read: None, structured_output: value, call_id: call_id.into(), status: if failed { @@ -2862,6 +2927,9 @@ mod skill_reminder_tests { file: "/repo/web/app.ts".into(), scope, root: root.clone(), + receipt: None, + operation_key: String::new(), + attempt: 1, }), }; match changed { @@ -2888,6 +2956,9 @@ mod skill_reminder_tests { file: "/repo/web/app.ts".into(), scope, root: root.clone(), + receipt: None, + operation_key: String::new(), + attempt: 1, }), ..run }; @@ -3107,6 +3178,9 @@ mod skill_reminder_tests { epoch: 1, }, root: Arc::new(cyber_core::rules::RootBinding::capture(root_dir.path()).unwrap()), + receipt: None, + operation_key: String::new(), + attempt: 1, }; let run = |outcome| ToolRun { call_id: "call".into(), @@ -3185,6 +3259,9 @@ mod skill_reminder_tests { epoch: 1, }, root: Arc::new(cyber_core::rules::RootBinding::capture(root_dir.path()).unwrap()), + receipt: None, + operation_key: String::new(), + attempt: 1, }), }; assert!(run.nested_target().is_none()); diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 33e0b1411..3341be836 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -366,6 +366,36 @@ pub struct ToolDispatched { pub operation_key: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RuleRead { + pub evidence: cyber_core::rules::ReadEvidence, + pub epoch: u32, + pub location: String, + pub operation_key: String, + pub attempt: u32, +} +impl RuleRead { + pub fn matches( + &self, + binding: &super::instructions::RuleBinding, + epoch: u32, + info: &super::model::SessionInfo, + call: &super::model::CallState, + ) -> bool { + call.status == super::model::CallStatus::Dispatched + && call.name.eq_ignore_ascii_case("read") + && self.epoch == epoch + && self.location == info.directory + && self.attempt != 0 + && self.attempt == call.attempt + && self.operation_key == format!("{}:{}", info.id, call.call_id) + && binding.native_root.as_deref() == Some(self.evidence.root_identity.as_str()) + && self.evidence.root.to_str() == Some(binding.root.as_str()) + && self.evidence.root.is_absolute() + && self.evidence.path.starts_with(&self.evidence.root) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ToolSettled { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -378,6 +408,8 @@ pub struct ToolSettled { /// Approved rule definitions actually labelled in this successful result. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub rule_reminders: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rule_read: Option, #[serde( default, skip_serializing_if = "Option::is_none", diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index dbcf6a66e..a25e5a838 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -488,6 +488,14 @@ pub struct SkillSuggestion { pub description: String, } +/// An execution with actual file reads, when the host instruments them. `None` +/// preserves legacy ordinary instruction loading; `Some([])` proves no eligible read. +#[derive(Debug)] +pub struct ObservedExecution { + pub outcome: ToolOutcome, + pub reads: Option>>, +} + /// How a tool run ended. #[derive(Debug, Clone, PartialEq)] pub enum ToolOutcome { @@ -1076,6 +1084,21 @@ pub trait ToolHost: Send + Sync { /// Run a call. Implementations stop within 2 s once `cancel` fires. fn execute(&self, call: Invocation, cancel: CancellationToken) -> BoxFuture<'_, ToolOutcome>; + /// Run with native read observations when the host can supply them. The + /// default has no operation receipt and cannot activate scoped rules. + fn execute_observed( + &self, + invocation: Invocation, + cancel: CancellationToken, + ) -> BoxFuture<'_, ObservedExecution> { + Box::pin(async move { + ObservedExecution { + outcome: self.execute(invocation, cancel).await, + reads: None, + } + }) + } + /// An explicit user request for a forked background child. Deny rules still apply. fn subtask( &self, diff --git a/crates/cyber-server/src/runtime/instructions.rs b/crates/cyber-server/src/runtime/instructions.rs index 6ffd4793a..2aec5613a 100644 --- a/crates/cyber-server/src/runtime/instructions.rs +++ b/crates/cyber-server/src/runtime/instructions.rs @@ -53,6 +53,8 @@ pub struct InstructionSettings { /// for hosts without a permission policy. pub read_gate: Option, pub rule_gate: Option, + pub scoped_rules: bool, + pub native_root: Option, } #[derive(Debug, Clone)] @@ -81,12 +83,22 @@ pub struct RuleBinding { pub path: String, pub digest: String, pub generation: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_root: Option, } impl RuleBinding { pub fn identity(&self) -> String { use sha2::{Digest, Sha256}; let mut hash = Sha256::new(); - hash.update(b"cyber-rule-reminder-v1\0"); + hash.update(if self.native_root.is_some() { + b"cyber-rule-reminder-v2\0" + } else { + b"cyber-rule-reminder-v1\0" + }); + if let Some(root) = &self.native_root { + hash.update((root.len() as u64).to_le_bytes()); + hash.update(root.as_bytes()); + } for field in [&self.root, &self.path, &self.digest, &self.generation] { hash.update((field.len() as u64).to_le_bytes()); hash.update(field.as_bytes()); @@ -178,6 +190,8 @@ impl InstructionSettings { home: cyber_core::paths::home_dir(env), read_gate: None, rule_gate: None, + scoped_rules: true, + native_root: None, } } @@ -590,7 +604,9 @@ impl<'a> Loader<'a> { { return Err("rule definition differs from its approved digest".into()); } - if !definition.applies(root, target) { + if (definition.paths.is_some() && !self.settings.scoped_rules) + || !definition.applies(root, target) + { return Ok(()); } let generation = authorization @@ -601,6 +617,7 @@ impl<'a> Loader<'a> { path: path.to_str().ok_or("rule path must be UTF-8")?.into(), digest: definition.digest.clone(), generation: generation.into(), + native_root: self.settings.native_root.clone(), }; let identity = binding.identity(); if !self.seen.insert(identity.clone()) { diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 7f0814cbe..553ddd93c 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -252,10 +252,10 @@ pub use host::{ AgentInference, CatalogResolver, CheckOutcome, ChildContinuation, ContentSource, FileDecision, FileDiff, FileReadError, GoalWorktree, HookEffects, Invocation, LifecycleHook, LifecycleVerdict, LocationGuard, LocationLease, ModelResolver, NoSnapshots, NoTools, - Provenance, Reconciliation, ResolvedModel, RestoreError, RuleEffect, RunWorktree, - SessionEnvelope, SkillSuggestion, Snapshot, SnapshotLocationGuard, SnapshotRecovery, Snapshots, - TeamWorktree, ToolDef, ToolHost, ToolOutcome, ToolScope, TurnContext, TurnWorktree, - WorkflowAgentOutput, WorkflowAgentRequest, WorkflowToolRequest, + ObservedExecution, Provenance, Reconciliation, ResolvedModel, RestoreError, RuleEffect, + RunWorktree, SessionEnvelope, SkillSuggestion, Snapshot, SnapshotLocationGuard, + SnapshotRecovery, Snapshots, TeamWorktree, ToolDef, ToolHost, ToolOutcome, ToolScope, + TurnContext, TurnWorktree, WorkflowAgentOutput, WorkflowAgentRequest, WorkflowToolRequest, }; pub use model::{ AssistantEntry, CallState, CallStatus, Delivery, Entry, InboxRow, InputStatus, LastStep, @@ -2317,6 +2317,7 @@ impl Runtime { skill_reminders: Vec::new(), instruction_reminders: Vec::new(), rule_reminders: Vec::new(), + rule_read: None, structured_output: None, call_id: call_id.into(), status, diff --git a/crates/cyber-server/src/runtime/model.rs b/crates/cyber-server/src/runtime/model.rs index ce50b5182..6cc89b1ed 100644 --- a/crates/cyber-server/src/runtime/model.rs +++ b/crates/cyber-server/src/runtime/model.rs @@ -1512,6 +1512,14 @@ impl SessionState { epoch.reminded_rules.extend( s.rule_reminders .iter() + .filter(|binding| match (&binding.native_root, &s.rule_read) { + (None, None) => true, // historical, path-only binding + (Some(_), Some(read)) => { + !self.epoch_stale + && read.matches(binding, epoch.number, &self.info, call) + } + _ => false, + }) .map(super::instructions::RuleBinding::identity), ); } diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index 8198c8e12..096c9b263 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1602,89 +1602,25 @@ impl ToolHost for BuiltinHost { } fn execute(&self, inv: Invocation, cancel: CancellationToken) -> BoxFuture<'_, ToolOutcome> { + self.execute_inner(inv, cancel, Default::default()) + } + + fn execute_observed( + &self, + inv: Invocation, + cancel: CancellationToken, + ) -> BoxFuture<'_, cyber_server::runtime::ObservedExecution> { Box::pin(async move { - if inv.name.starts_with("mcp__") { - return match self.browser_binding(&inv) { - Some(binding) => self.execute_browser(inv, binding, cancel).await, - None => self.execute_mcp(inv, cancel).await, - }; - } - let mut inv = inv; - if let Err(error) = inv.asker.validate_auto_override(&inv.name, &inv.input) { - return ToolOutcome::Failed(error.to_string()); - } - let Some(tool) = self.tools.iter().find(|t| t.def().spec.name == inv.name) else { - if let Some(offered) = self.plugin_tool(&inv) { - return self.execute_plugin(inv, offered, cancel).await; - } - return ToolOutcome::Failed(format!("Unknown tool: {}", inv.name)); - }; - if let Err(error) = self.check_skill_tool(&inv).await { - return ToolOutcome::Failed(error); - } - if let Err(error) = self.check_agent_tool(&inv) { - return ToolOutcome::Failed(error); - } - if inv.name == "computer" && !crate::tools::computer::valid_input(&inv.input) { - return ToolOutcome::Failed("Invalid native computer action".into()); - } - if let Err(message) = crate::schema::validate(&tool.def().spec.input_schema, &inv.input) - { - return ToolOutcome::Failed(if inv.name == "computer" { - "Invalid native computer action".into() - } else { - message - }); - } - let hook_decision = match self - .pre_tool_hooks(&mut inv, &tool.def().spec.input_schema, cancel.clone()) - .await - { - Ok(allowed) => allowed, - Err(ToolError::Failed(error)) => return ToolOutcome::Failed(error), - Err(ToolError::Aborted) => return ToolOutcome::Aborted, - }; - let started = std::time::Instant::now(); - let ctx = match self - .call_ctx(&inv, hook_decision, cancel.clone(), DenialHooks::armed()) - .await - { - Ok(ctx) => ctx, - Err(error) => return ToolOutcome::Failed(error), - }; - let outcome = self.run_builtin(tool.as_ref(), &ctx).await; - if ctx.inv.name == "computer" { - return self.settle_call(&ctx, outcome, started).await; + let read_receipt = ReadReceiptSlot::default(); + let outcome = self.execute_inner(inv, cancel, read_receipt.clone()).await; + let read = read_receipt + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + cyber_server::runtime::ObservedExecution { + outcome, + reads: Some(read.into_iter().collect()), } - let Some(input) = ctx.denial.take_retry(&outcome) else { - return self.settle_call(&ctx, outcome, started).await; - }; - // A PermissionDenied retry is re-validated and re-evaluated exactly once; - // its denial is final. Changed input passes PreToolUse again first. - let schema = &tool.def().spec.input_schema; - let retried = self - .retried_invocation(&inv, schema, input, hook_decision, cancel.clone()) - .await; - let (retried, decision) = match retried { - Ok(retried) => retried, - Err(ToolError::Failed(error)) => { - return self - .settle_call(&ctx, ToolOutcome::Failed(error), started) - .await; - } - Err(ToolError::Aborted) => { - return self.settle_call(&ctx, ToolOutcome::Aborted, started).await; - } - }; - let ctx = match self - .call_ctx(&retried, decision, cancel, DenialHooks::default()) - .await - { - Ok(ctx) => ctx, - Err(error) => return ToolOutcome::Failed(error), - }; - let outcome = self.run_builtin(tool.as_ref(), &ctx).await; - self.settle_call(&ctx, outcome, started).await }) } @@ -1793,6 +1729,7 @@ impl ToolHost for BuiltinHost { self.check_agent_tool(&inv)?; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -1930,6 +1867,7 @@ impl BuiltinHost { ) -> Result, String> { Ok(Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -2021,12 +1959,17 @@ impl BuiltinHost { hook_decision: Option, cancel: CancellationToken, denial: DenialHooks, + read_receipt: ReadReceiptSlot, ) -> Result, String> { let policy = self.policy(inv).await?; if let Some(reason) = policy.skill_tool_denial(&inv.name) { return Err(reason); } + // A hook retry owns a fresh attempt's observation; never carry a prior + // read into a directory listing or an uninstrumented retried execution. + *read_receipt.lock().unwrap_or_else(PoisonError::into_inner) = None; Ok(Ctx { + read_receipt, skill_paths: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), @@ -2113,7 +2056,10 @@ impl BuiltinHost { } /// Everything a tool needs for one call. +pub(crate) type ReadReceiptSlot = Arc>>>; + pub(crate) struct Ctx<'a> { + pub read_receipt: ReadReceiptSlot, pub skill_paths: Mutex>, pub loaded_skill: Mutex>, /// Set by a tool whose result comes from outside the workspace; the projection @@ -2847,3 +2793,109 @@ impl BuiltinHost { } mod workflow_verify; + +impl BuiltinHost { + fn execute_inner( + &self, + inv: Invocation, + cancel: CancellationToken, + read_receipt: ReadReceiptSlot, + ) -> BoxFuture<'_, ToolOutcome> { + Box::pin(async move { + if inv.name.starts_with("mcp__") { + return match self.browser_binding(&inv) { + Some(binding) => self.execute_browser(inv, binding, cancel).await, + None => self.execute_mcp(inv, cancel).await, + }; + } + let mut inv = inv; + if let Err(error) = inv.asker.validate_auto_override(&inv.name, &inv.input) { + return ToolOutcome::Failed(error.to_string()); + } + let Some(tool) = self.tools.iter().find(|t| t.def().spec.name == inv.name) else { + if let Some(offered) = self.plugin_tool(&inv) { + return self.execute_plugin(inv, offered, cancel).await; + } + return ToolOutcome::Failed(format!("Unknown tool: {}", inv.name)); + }; + if let Err(error) = self.check_skill_tool(&inv).await { + return ToolOutcome::Failed(error); + } + if let Err(error) = self.check_agent_tool(&inv) { + return ToolOutcome::Failed(error); + } + if inv.name == "computer" && !crate::tools::computer::valid_input(&inv.input) { + return ToolOutcome::Failed("Invalid native computer action".into()); + } + if let Err(message) = crate::schema::validate(&tool.def().spec.input_schema, &inv.input) + { + return ToolOutcome::Failed(if inv.name == "computer" { + "Invalid native computer action".into() + } else { + message + }); + } + let hook_decision = match self + .pre_tool_hooks(&mut inv, &tool.def().spec.input_schema, cancel.clone()) + .await + { + Ok(allowed) => allowed, + Err(ToolError::Failed(error)) => return ToolOutcome::Failed(error), + Err(ToolError::Aborted) => return ToolOutcome::Aborted, + }; + let started = std::time::Instant::now(); + let ctx = match self + .call_ctx( + &inv, + hook_decision, + cancel.clone(), + DenialHooks::armed(), + read_receipt.clone(), + ) + .await + { + Ok(ctx) => ctx, + Err(error) => return ToolOutcome::Failed(error), + }; + let outcome = self.run_builtin(tool.as_ref(), &ctx).await; + if ctx.inv.name == "computer" { + return self.settle_call(&ctx, outcome, started).await; + } + let Some(input) = ctx.denial.take_retry(&outcome) else { + return self.settle_call(&ctx, outcome, started).await; + }; + // A PermissionDenied retry is re-validated and re-evaluated exactly once; + // its denial is final. Changed input passes PreToolUse again first. + let schema = &tool.def().spec.input_schema; + let retried = self + .retried_invocation(&inv, schema, input, hook_decision, cancel.clone()) + .await; + let (retried, decision) = match retried { + Ok(retried) => retried, + Err(ToolError::Failed(error)) => { + return self + .settle_call(&ctx, ToolOutcome::Failed(error), started) + .await; + } + Err(ToolError::Aborted) => { + return self.settle_call(&ctx, ToolOutcome::Aborted, started).await; + } + }; + let ctx = match self + .call_ctx( + &retried, + decision, + cancel, + DenialHooks::default(), + read_receipt.clone(), + ) + .await + { + Ok(ctx) => ctx, + Err(error) => return ToolOutcome::Failed(error), + }; + let outcome = self.run_builtin(tool.as_ref(), &ctx).await; + self.settle_call(&ctx, outcome, started).await + }) + } +} diff --git a/crates/cyber-tools/src/host/attachments.rs b/crates/cyber-tools/src/host/attachments.rs index d4e9eb17e..d327c0ef0 100644 --- a/crates/cyber-tools/src/host/attachments.rs +++ b/crates/cyber-tools/src/host/attachments.rs @@ -53,7 +53,13 @@ impl BuiltinHost { .await .map_err(denied)?; let mut ctx = self - .call_ctx(&invocation, hook_decision, cancel, DenialHooks::default()) + .call_ctx( + &invocation, + hook_decision, + cancel, + DenialHooks::default(), + Default::default(), + ) .await .map_err(FileReadError::Denied)?; ctx.location = super::canonical(Path::new(&invocation.directory)); diff --git a/crates/cyber-tools/src/host/command_skills.rs b/crates/cyber-tools/src/host/command_skills.rs index a66a292bd..d0503ebd3 100644 --- a/crates/cyber-tools/src/host/command_skills.rs +++ b/crates/cyber-tools/src/host/command_skills.rs @@ -237,6 +237,7 @@ impl BuiltinHost { // Withholding follows the reviewer's own ceiling, not the invoker's mode. super::Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/host/files.rs b/crates/cyber-tools/src/host/files.rs index 8be3c9024..b136e8974 100644 --- a/crates/cyber-tools/src/host/files.rs +++ b/crates/cyber-tools/src/host/files.rs @@ -106,6 +106,7 @@ impl BuiltinHost { ) -> Result, String> { Ok(Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/host/template_expand.rs b/crates/cyber-tools/src/host/template_expand.rs index bbb486a3c..15dbade44 100644 --- a/crates/cyber-tools/src/host/template_expand.rs +++ b/crates/cyber-tools/src/host/template_expand.rs @@ -185,6 +185,7 @@ impl BuiltinHost { ) -> Result, String> { Ok(Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/host/workflow_agents.rs b/crates/cyber-tools/src/host/workflow_agents.rs index 857a00b9f..a15a33443 100644 --- a/crates/cyber-tools/src/host/workflow_agents.rs +++ b/crates/cyber-tools/src/host/workflow_agents.rs @@ -88,6 +88,7 @@ impl BuiltinHost { self.check_agent_tool(&inv).map_err(denied)?; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -150,6 +151,7 @@ impl BuiltinHost { }; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/host/workflow_integration.rs b/crates/cyber-tools/src/host/workflow_integration.rs index 7e88051d2..ecc00d4e5 100644 --- a/crates/cyber-tools/src/host/workflow_integration.rs +++ b/crates/cyber-tools/src/host/workflow_integration.rs @@ -227,6 +227,7 @@ impl BuiltinHost { async fn integration_context<'a>(&'a self, inv: &'a Invocation) -> Result, String> { Ok(Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -297,6 +298,7 @@ impl BuiltinHost { } let context = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -621,6 +623,7 @@ impl BuiltinHost { policy.location = managed.path.clone(); let check_ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/host/workflow_solve.rs b/crates/cyber-tools/src/host/workflow_solve.rs index 823bd3a2f..d225da94b 100644 --- a/crates/cyber-tools/src/host/workflow_solve.rs +++ b/crates/cyber-tools/src/host/workflow_solve.rs @@ -93,6 +93,7 @@ impl BuiltinHost { let policy = self.policy(&inv).await.map_err(PrepareError::Unavailable)?; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/mcp/sampler.rs b/crates/cyber-tools/src/mcp/sampler.rs index c3f3ff432..14f2dce6b 100644 --- a/crates/cyber-tools/src/mcp/sampler.rs +++ b/crates/cyber-tools/src/mcp/sampler.rs @@ -134,6 +134,7 @@ async fn execute( .map_err(|_| "Sampling permission policy is unavailable")?; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/registered.rs b/crates/cyber-tools/src/registered.rs index 8841353cc..6e6b989aa 100644 --- a/crates/cyber-tools/src/registered.rs +++ b/crates/cyber-tools/src/registered.rs @@ -231,6 +231,7 @@ impl BuiltinHost { .insert(index, Rule::new(&inv.name, "*", effect, "default")); Ok(Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/sandboxing.rs b/crates/cyber-tools/src/sandboxing.rs index b73309eea..dc3091918 100644 --- a/crates/cyber-tools/src/sandboxing.rs +++ b/crates/cyber-tools/src/sandboxing.rs @@ -949,6 +949,7 @@ mod tests { }; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/tools/agent.rs b/crates/cyber-tools/src/tools/agent.rs index 9fc3d6dd4..9f1bf1856 100644 --- a/crates/cyber-tools/src/tools/agent.rs +++ b/crates/cyber-tools/src/tools/agent.rs @@ -392,6 +392,7 @@ pub(crate) async fn fork_skill( ctx.host.check_skill_tool(&inv).await.map_err(failed)?; let child_ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/tools/binary/patch.rs b/crates/cyber-tools/src/tools/binary/patch.rs index 177fd0da3..296b7f449 100644 --- a/crates/cyber-tools/src/tools/binary/patch.rs +++ b/crates/cyber-tools/src/tools/binary/patch.rs @@ -181,6 +181,7 @@ pub(super) async fn run( let policy = host.policy(&inv).await.map_err(|_| Error::Changed)?; let fresh = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/tools/fs.rs b/crates/cyber-tools/src/tools/fs.rs index 02fcf1b61..0280c1631 100644 --- a/crates/cyber-tools/src/tools/fs.rs +++ b/crates/cyber-tools/src/tools/fs.rs @@ -34,6 +34,8 @@ impl Tool for Read { fn run<'a>(&'a self, ctx: &'a Ctx<'a>) -> BoxFuture<'a, Result> { Box::pin(async move { + let declared = ctx.resolve(text(&ctx.inv.input, "path")); + let authorized = declared.canonicalize().ok(); let path = readable(ctx).await?; // A managed overflow file is re-read in its original result's frame. ctx.frame_overflow_reads(&path); @@ -48,9 +50,24 @@ impl Tool for Read { .lsp .get() .and_then(|_| crate::lsp::read_origin(&ctx.location, &path).ok()); - let bytes = tokio::fs::read(&path) - .await - .map_err(|e| failed(format!("Could not read {}: {e}", path.display())))?; + let reader = authorized + .as_deref() + .and_then(|target| observed_reader(ctx, &path, target)); + let (bytes, receipt) = match reader { + Some(reader) => { + let (bytes, receipt) = tokio::task::spawn_blocking(move || reader.read()) + .await + .map_err(|_| failed("Observed file read failed"))? + .map_err(|e| failed(format!("Could not read {}: {e}", path.display())))?; + (bytes, Some(std::sync::Arc::new(receipt))) + } + None => ( + tokio::fs::read(&path) + .await + .map_err(|e| failed(format!("Could not read {}: {e}", path.display())))?, + None, + ), + }; if let Some(kind) = media_kind(&path) { return Err(failed(format!( "{kind} files are returned as media from a later milestone; this build reads text only" @@ -59,6 +76,9 @@ impl Tool for Read { if is_binary(&bytes) { return Err(failed("Binary file: application/octet-stream")); } + *ctx.read_receipt + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = receipt; ctx.host.mark_read(&ctx.inv.session_id, &path); ctx.note_skill_path(&path); if let Some(locations) = ctx.host.lsp.get() @@ -86,6 +106,37 @@ impl Tool for Read { } } +/// An alias may have a different policy resource from the physical target. +/// An authorization of the spelling alone cannot attest a denied or unreviewed +/// canonical target. Withhold the receipt without changing the read contract. +fn observed_reader( + ctx: &Ctx<'_>, + path: &Path, + authorized: &Path, +) -> Option { + let location = ctx.location.canonicalize().ok()?; + let physical_resource = authorized.strip_prefix(&location).map_or_else( + |_| ctx.resource(authorized), + |relative| ctx.resource(&ctx.location.join(relative)), + ); + if physical_resource != ctx.resource(path) { + let request = Request { + tool: Some("read".into()), + action: "read".into(), + resources: vec![physical_resource], + read_only: true, + ..Request::default() + }; + if ctx.policy.decide(&request) != crate::permissions::Decision::Allow + || ctx.policy.taint_holds(&request) + || ctx.policy.forces_classifier(&request) + { + return None; + } + } + cyber_core::rules::ObservedRead::open(&ctx.location, path, authorized).ok() +} + /// A page of the file redacted as a whole first, so a secret that the page boundary /// cuts (a private key) is still hidden; markers keep line breaks, so line numbers are /// the file's. What the page hides is noted for the settlement. diff --git a/crates/cyber-tools/src/worktrees.rs b/crates/cyber-tools/src/worktrees.rs index 182f3161a..9cb9f89e2 100644 --- a/crates/cyber-tools/src/worktrees.rs +++ b/crates/cyber-tools/src/worktrees.rs @@ -109,6 +109,7 @@ impl BuiltinHost { }; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -174,6 +175,7 @@ impl BuiltinHost { }; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -377,6 +379,7 @@ impl BuiltinHost { } let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -469,6 +472,7 @@ impl BuiltinHost { ) -> io::Result> { let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -710,6 +714,7 @@ impl BuiltinHost { ) -> io::Result> { Ok(Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -977,6 +982,7 @@ impl BuiltinHost { } let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/worktrees/child.rs b/crates/cyber-tools/src/worktrees/child.rs index 6356a9fa0..2e018a619 100644 --- a/crates/cyber-tools/src/worktrees/child.rs +++ b/crates/cyber-tools/src/worktrees/child.rs @@ -812,6 +812,7 @@ impl ChildWorktree { }; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -876,6 +877,7 @@ impl ChildWorktree { } let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/crates/cyber-tools/src/worktrees/recovery.rs b/crates/cyber-tools/src/worktrees/recovery.rs index 3f7804b5a..f860929c5 100644 --- a/crates/cyber-tools/src/worktrees/recovery.rs +++ b/crates/cyber-tools/src/worktrees/recovery.rs @@ -50,6 +50,7 @@ impl BuiltinHost { }; let ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), @@ -188,6 +189,7 @@ impl BuiltinHost { child_inv.rules = context.child.rules.clone(); let child_ctx = Ctx { skill_paths: Default::default(), + read_receipt: Default::default(), loaded_skill: Default::default(), provenance: Default::default(), compiler_feedback: Default::default(), diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index e7a6daf02..359ab17e1 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Nested read delivery now reloads settings after success and checks captured Location/Epoch plus native Location ancestry and project boundary before/after discovery and under the settlement lock. Finish edit/patch/partial-effect delivery, native operation/config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native read and dispatched attempt. Finish edit/patch/partial-effect delivery, exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 1b33a02fe..423233c67 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1221,7 +1221,9 @@ Project rule bytes are included in the existing checkout trust digest. Review wi Definition reads use the fenced native reader: linked files/directories, hard links and special files are refused. The reader caps a definition at 1 MiB; frontmatter is limited to 16 KiB, with up to 64 relative patterns of 256 bytes each. A project inventory covers up to 128 definitions and 4 MiB, with bounded directory enumeration. These review/discovery bounds are separate from the smaller rendered instruction budgets. Unknown inventories grant no rule authority. `cyber doctor` reports held, refused and truncated baseline rules. -The current local implementation activates scoped rules after successful built-in reads and records typed root/path/digest/approval bindings separately from legacy instruction paths. Nested discovery resolves settings after the successful read and checks that its captured Location and Context Epoch still apply before loading and before settlement. The read retains the physical Location's native directory identities and withholds reminders after root/ancestor replacement, removal, link substitution or a changed project boundary. A suppressed reminder remains eligible for a later operation. These checks do not yet bind the actual file handle used by the operation. Edit/patch and partial-settlement delivery, native operation and exact configuration-generation binding, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. +Scoped rules activate after a successful built-in file read with a native receipt. The tool matches the physical file it actually opened after authorization and any hook rewrite. A directory listing, unsupported observation, or changed root/target binding does not consume delivery. The tool's own result is preserved. Ordinary instruction behavior remains available to legacy hosts without native receipts; those hosts cannot activate path-scoped rules. + +The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Edit/patch and partial-settlement delivery, exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 4a0695b35..6b0284e93 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -28,6 +28,8 @@ Definition discovery bounds are 128 project files, 4096 examined entries per rul Nested read discovery resolves instruction settings after a successful effect. Its captured Session Location and Context Epoch must still be active before discovery and again under the settlement lock. An absent/stale/replaced Epoch or changed Location suppresses the reminder without recording delivery; normal group settlement and exact `/approve` replay share this check. The captured read also retains the native directory identities from the volume root through the physical Location, verifies the project-root boundary, and rechecks those bindings before/after discovery and under settlement. Unix retains directory handles to prevent inode reuse; Windows releases handles after capture and verifies full native IDs, so the observation does not freeze editor/worktree renames. Ordinary file content edits do not invalidate the root. Replacement/removal, changed ancestors, linked substitution, a retargeted declared Location alias or a changed project boundary withhold the reminder. These point-in-time context/root checks do not prove which file handle performed the effect: actual per-file operation receipts (including partial writes), exact configuration-generation fencing through commit and revocation withdrawal remain required. +Built-in reads now read through an observed file handle after authorization and any input rewrite, and hand the resulting receipt to the runtime through the host execution boundary. Workspace read targets preserve the normal size and hard-link behavior; rule-definition and memory-storage source readers keep their stricter checks. The receipt retains actual native file/ancestor identities, a physical path and the captured project root. Unsupported observations keep the read result but grant no scoped-rule delivery. Rechecking a receipt reads metadata, not file contents. The ordinary legacy host path cannot activate scoped definitions. Successful labelled delivery stores `rule_read` atomically with `rule_reminders`: actual target/root IDs, captured Location/Epoch, operation key and dispatched attempt. Version-2 rule IDs incorporate the native root; old IDs and legacy records replay unchanged. The recorded proof describes an earlier operation and never grants new filesystem authority. Replay checks the dispatch and context before folding new identities. Actual edit/patch receipts, positive partial effects, exact configuration generation and revocation/restart acceptance remain open. + Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. ### 4. Root structured output is per admitted prompt diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index cfb5b8160..d98871259 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -68,3 +68,11 @@ - **THEN** same-path replacement sources SHALL NOT supply reminders for the old operation - **AND** the operation result SHALL remain intact and its suppressed reminder SHALL NOT consume delivery in the active Epoch - **AND** a later operation captured against the replacement Location SHALL remain independently eligible + +#### Scenario: A hook rewrites the authorized read target +- **GIVEN** a PreToolUse hook rewrites a read of a web file to an authorized database file +- **WHEN** the built-in tool reads through its native file handle +- **THEN** scoped rule matching SHALL use the actual physical database target +- **AND** the settlement SHALL bind any delivered rule to that read's native root/file identities, Location, Context Epoch, operation key and dispatched attempt +- **AND** a listing, denied operation or successful result without eligible native evidence SHALL NOT consume scoped rule delivery +- **AND** replay SHALL NOT consume a new rule identity from a receipt mismatched to the dispatch, root, Location or Epoch diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index c96a05a78..28bccd9b0 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1752,3 +1752,48 @@ include the shared native snapshot helper; its existing rules/context/App steps exercise the new cases. Current-source native execution and whole-goal review remain required before delivery; no goal/task acceptance is inferred from these local checks. + + +Native read receipt continuation (2026-11-10): built-in file reads carry the +receipt of their actual open handle through a typed ToolHost/App execution port. +The pre-authorization physical target must still match at open. An alias with a +separate non-allow/taint/classifier decision cannot attest the canonical target. +Actual workspace reads preserve their size and hard-link behavior; strict rule +and memory definition readers keep existing link restrictions. Read fallback, +listings, denied/failed results and unsupported receipts cannot activate scoped +rules. The capture uses actual post-hook input, not the provider's original path. +The receipt revalidates native root, file and ancestor IDs and declared aliases +before/after discovery and under settlement. A successful `rule_read` record +binds native evidence, Location/Epoch, dispatched attempt and operation key to +the rule reminders in the same Store event. New rule IDs incorporate the native +root; historical path-only IDs remain compatible. Replay refuses to consume a +new identity on a mismatched dispatch, Location, Epoch or root. The native +metadata describes the past read and grants no future filesystem authority. + +Review fixed receipt reuse across host retries, separated workspace hard-link +observation from strict definition/storage readers, required UTF-8 receipt paths, +and fenced the authorization-time physical target. Actual App hook-rewrite +acceptance verifies only the database rule reaches the provider after a web read +is rewritten, while the original call input remains in the transcript. Store +replay checks Location/Epoch/attempt/key mismatch, and Core tests exercise actual +handle bytes, alias retargeting and large/hard-linked targets. This is read +provenance; edit/patch/partial-effect receipts, exact config generation through +commit, rule withdrawal, full restart and inspection still prevent completion +of 4.1–4.4. No parent task closes. + +GitHub native run [42890123485](https://github.com/CyberdyneCorp/cybercode-cli/actions/runs/42890123485) +passed at `c9e0c373`. The later `fa2c9dfb` and `25e7e89f` runs were cancelled; +those results do not establish native acceptance of their changes or the new +receipt code. Main is fully integrated at `8266db6c` after fetch. + +Local receipt verification: 11 filtered Core tests, nine import snapshot tests, +455 server library tests (one ignored subprocess probe), 22 context tests, 16 App +instruction tests (the command-hook rewrite fixture is Unix), and the AppHost +forwarding check passed. Core/Server/Tools/App all-target Clippy with `-D warnings`, +formatting/diff, workflow YAML, generated SDK consistency and pinned strict +OpenSpec (100 items) passed; spec lint reports zero errors and 25 warnings. +Native C/linker warnings were observed without an origin claim. The shared native +reader's import tests are included in the platform workflow. Windows, whole +workspace, SDK runtime and current-source native matrix acceptance remain unrun +locally for this increment. Checkpoint review found no remaining findings in the +read changes after the fixes above; full-goal review remains open. From 856f522b7c6b2d8929964f02ef32da4f32568667 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 18:37:53 -0400 Subject: [PATCH 80/87] Record native receipts for scoped edit rules --- crates/cyber-app/src/host.rs | 2 +- crates/cyber-app/tests/instructions.rs | 52 +++++++++ crates/cyber-core/src/import/mod.rs | 3 +- crates/cyber-core/src/import/snapshot.rs | 41 ++++--- crates/cyber-core/src/rules.rs | 101 +++++++++++++----- .../cyber-server/src/runtime/auto_override.rs | 10 +- crates/cyber-server/src/runtime/drain.rs | 75 ++++++++----- crates/cyber-server/src/runtime/events.rs | 13 ++- crates/cyber-server/src/runtime/host.rs | 10 +- crates/cyber-server/src/runtime/mod.rs | 1 + crates/cyber-server/src/runtime/model.rs | 30 ++++-- crates/cyber-tools/src/host.rs | 12 +-- crates/cyber-tools/src/tools/fs.rs | 49 ++++++++- docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 4 +- .../complete-daily-coding-surfaces/design.md | 2 +- .../complete-daily-coding-surfaces/tasks.md | 15 +++ 17 files changed, 317 insertions(+), 105 deletions(-) diff --git a/crates/cyber-app/src/host.rs b/crates/cyber-app/src/host.rs index 9fac4765b..5c64ebbd0 100644 --- a/crates/cyber-app/src/host.rs +++ b/crates/cyber-app/src/host.rs @@ -374,7 +374,7 @@ impl ToolHost for AppHost { return Box::pin(async move { cyber_server::runtime::ObservedExecution { outcome: self.execute(call, cancel).await, - reads: None, + files: None, } }); } diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index b0f7cfffa..7e5e768e0 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -28,6 +28,11 @@ fn reply(body: &Value) -> Value { return json!({"content": "done"}); } let text = last.to_string(); + if text.contains("edit the app") { + let arguments = json!({"path":"web/src/app.ts","old_string":"const version = 1","new_string":"const version = 2"}).to_string(); + return json!({"tool_calls":[{"index":0,"id":"call_edit","type":"function", + "function":{"name":"edit","arguments":arguments}}]}); + } for (prompt, id) in [ ("read the app first", "call_first"), ("read the app again", "call_again"), @@ -896,3 +901,50 @@ async fn rewritten_reads_deliver_only_rules_for_the_file_actually_opened() { assert_eq!(state.epoch.unwrap().reminded_rules.len(), 1); f.app.runtime.shutdown().await; } + +#[tokio::test] +async fn denied_read_leaves_rule_available_for_a_successful_edit() { + let f = Fixture::start_with(json!({"permissions":{"read":{"web/src/app.ts":"deny"}}})).await; + f.write("web/src/app.ts", "const version = 1"); + f.write( + ".cyber/rules/web.md", + "---\npaths: ['web/**']\n---\nEDIT_RULE_GUIDANCE", + ); + f.approve(&f.repo); + let id = f.session(&f.repo).await; + f.prompt(&id, &f.repo, "read the app first", 2).await; + let before = f.app.runtime.state(&id).await.unwrap(); + assert!( + before + .calls + .values() + .any(|call| call.name == "read" + && call.status == cyber_server::runtime::CallStatus::Error) + ); + assert!(before.epoch.as_ref().unwrap().reminded_rules.is_empty()); + assert!(!last_tool_result(&f.last_request()).contains("EDIT_RULE_GUIDANCE")); + + f.prompt(&id, &f.repo, "edit the app", 2).await; + assert_eq!( + std::fs::read_to_string(f.repo.join("web/src/app.ts")).unwrap(), + "const version = 2" + ); + assert!(last_tool_result(&f.last_request()).contains("EDIT_RULE_GUIDANCE")); + let state = f.app.runtime.state(&id).await.unwrap(); + let epoch = state.epoch.unwrap(); + assert_eq!(epoch.number, before.epoch.unwrap().number); + assert_eq!(epoch.reminded_rules.len(), 1); + let events = f.app.store.read_events(&id, -1, 500).unwrap().events; + let settled = events + .iter() + .find(|event| event.data["rule_edit"].is_object()) + .unwrap(); + assert!(settled.data.get("rule_read").is_none()); + assert_eq!( + settled.data["rule_edit"]["evidence"]["path"], + f.repo.join("web/src/app.ts").to_str().unwrap() + ); + let replay = cyber_server::runtime::SessionState::replay(&events).unwrap(); + assert_eq!(replay.epoch.unwrap().reminded_rules, epoch.reminded_rules); + f.app.runtime.shutdown().await; +} diff --git a/crates/cyber-core/src/import/mod.rs b/crates/cyber-core/src/import/mod.rs index ce1bb6d84..3078a653a 100644 --- a/crates/cyber-core/src/import/mod.rs +++ b/crates/cyber-core/src/import/mod.rs @@ -46,7 +46,8 @@ pub use preview::{ pub use provenance::SourceReference; pub use snapshot::SourceSnapshot; pub(crate) use snapshot::{ - DirectorySnapshot, FileRead, FileReadReceipt, command_directory, verify_command_directory, + DirectorySnapshot, FileHandleReceipt, ObservedFileHandle, command_directory, + verify_command_directory, }; mod opencode_providers; diff --git a/crates/cyber-core/src/import/snapshot.rs b/crates/cyber-core/src/import/snapshot.rs index 68244ad12..3aa43c1c0 100644 --- a/crates/cyber-core/src/import/snapshot.rs +++ b/crates/cyber-core/src/import/snapshot.rs @@ -202,26 +202,42 @@ impl DirectorySnapshot { /// An actual read performed through the captured no-follow file handle. Unlike /// definition snapshots this preserves the read tool's existing file-size contract. -pub(crate) struct FileRead { +pub(crate) struct ObservedFileHandle { path: PathBuf, opened: OpenSource, } -pub(crate) struct FileReadReceipt { +pub(crate) struct FileHandleReceipt { path: PathBuf, bindings: Vec, state: FileState, _retained: Option, } -impl FileRead { +impl ObservedFileHandle { pub(crate) fn open(path: &Path) -> Result { Ok(Self { path: path.into(), opened: OpenSource::open_observed(path, true)?, }) } - pub(crate) fn read(mut self) -> std::io::Result<(Vec, FileReadReceipt)> { + pub(crate) fn read(mut self) -> std::io::Result<(Vec, FileHandleReceipt)> { let mut bytes = Vec::new(); self.opened.file.read_to_end(&mut bytes)?; + Ok((bytes, self.receipt())) + } + + pub(crate) fn after_write(self, file: &File) -> Result { + if observed_identity(file, &self.path, true)? != self.opened.state.identity { + return Err(error( + &self.path, + "write handle differs from the authorized file", + )); + } + let receipt = self.receipt(); + receipt.verify()?; + Ok(receipt) + } + + fn receipt(self) -> FileHandleReceipt { let bindings = self.opened.bindings.clone(); let state = self.opened.state.clone(); #[cfg(windows)] @@ -231,18 +247,15 @@ impl FileRead { }; #[cfg(not(windows))] let retained = Some(self.opened); - Ok(( - bytes, - FileReadReceipt { - path: self.path, - bindings, - state, - _retained: retained, - }, - )) + FileHandleReceipt { + path: self.path, + bindings, + state, + _retained: retained, + } } } -impl FileReadReceipt { +impl FileHandleReceipt { pub(crate) fn verify(&self) -> Result<(), DiscoveryError> { let current = OpenSource::open_observed(&self.path, true)?; if current.bindings != self.bindings || current.state.identity != self.state.identity { diff --git a/crates/cyber-core/src/rules.rs b/crates/cyber-core/src/rules.rs index c8224bb63..11acb8652 100644 --- a/crates/cyber-core/src/rules.rs +++ b/crates/cyber-core/src/rules.rs @@ -62,23 +62,23 @@ impl std::fmt::Debug for RootBinding { /// Metadata describing a successful read; these hashes are observations, never /// authority to perform another operation or reopen a source after restart. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct ReadEvidence { +pub struct FileEvidence { pub root: PathBuf, pub path: PathBuf, pub root_identity: String, pub file_identity: String, } -pub struct ObservedRead { +pub struct ObservedFile { declared: PathBuf, physical: PathBuf, root: RootBinding, - reader: crate::import::FileRead, + reader: crate::import::ObservedFileHandle, } -pub struct ReadReceipt { +pub struct FileReceipt { declared: PathBuf, root: RootBinding, - native: crate::import::FileReadReceipt, - evidence: ReadEvidence, + native: crate::import::FileHandleReceipt, + evidence: FileEvidence, } impl RootBinding { pub fn identity(&self) -> String { @@ -87,7 +87,7 @@ impl RootBinding { .expect("project root is an ancestor") } } -impl ObservedRead { +impl ObservedFile { pub fn open(location: &Path, path: &Path, authorized: &Path) -> Result { let root = RootBinding::capture(location)?; let declared = std::path::absolute(path).map_err(|_| "read path is invalid")?; @@ -104,7 +104,8 @@ impl ObservedRead { if physical.to_str().is_none() || root.root().to_str().is_none() { return Err("read receipt paths must be UTF-8".into()); } - let reader = crate::import::FileRead::open(&physical).map_err(|e| e.reason.to_owned())?; + let reader = + crate::import::ObservedFileHandle::open(&physical).map_err(|e| e.reason.to_owned())?; root.verify()?; Ok(Self { declared, @@ -113,27 +114,48 @@ impl ObservedRead { reader, }) } - pub fn read(self) -> std::io::Result<(Vec, ReadReceipt)> { + pub fn read(self) -> std::io::Result<(Vec, FileReceipt)> { let (bytes, native) = self.reader.read()?; - let evidence = ReadEvidence { - root: self.root.root().into(), - path: self.physical, - root_identity: self.root.identity(), - file_identity: native.identity(), - }; Ok(( bytes, - ReadReceipt { - declared: self.declared, - root: self.root, - native, - evidence, - }, + FileReceipt::from_native(self.declared, self.physical, self.root, native), )) } + + /// Bind a completed write to the file handle that performed it. + pub fn after_write(self, file: &std::fs::File) -> Result { + let native = self + .reader + .after_write(file) + .map_err(|error| error.reason.to_owned())?; + let receipt = FileReceipt::from_native(self.declared, self.physical, self.root, native); + receipt.verify()?; + Ok(receipt) + } } -impl ReadReceipt { - pub fn evidence(&self) -> &ReadEvidence { + +impl FileReceipt { + fn from_native( + declared: PathBuf, + physical: PathBuf, + root: RootBinding, + native: crate::import::FileHandleReceipt, + ) -> Self { + let evidence = FileEvidence { + root: root.root().into(), + path: physical, + root_identity: root.identity(), + file_identity: native.identity(), + }; + Self { + declared, + root, + native, + evidence, + } + } + + pub fn evidence(&self) -> &FileEvidence { &self.evidence } pub fn verify(&self) -> Result<(), String> { @@ -144,9 +166,9 @@ impl ReadReceipt { self.native.verify().map_err(|e| e.reason.to_owned()) } } -impl std::fmt::Debug for ReadReceipt { +impl std::fmt::Debug for FileReceipt { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ReadReceipt") + f.debug_struct("FileReceipt") .field("evidence", &self.evidence) .finish_non_exhaustive() } @@ -500,7 +522,7 @@ mod tests { let path = root.join("large.txt"); let content = vec![b'x'; 1024 * 1024 + 1]; std::fs::write(&path, &content).unwrap(); - let reader = ObservedRead::open(&root, &path, &path.canonicalize().unwrap()).unwrap(); + let reader = ObservedFile::open(&root, &path, &path.canonicalize().unwrap()).unwrap(); let (bytes, receipt) = reader.read().unwrap(); assert_eq!(bytes, content); assert!(receipt.verify().is_ok()); @@ -528,7 +550,7 @@ mod tests { std::fs::write(&path, "workspace content").unwrap(); std::fs::hard_link(&path, &link).unwrap(); let (bytes, receipt) = - ObservedRead::open(temp.path(), &link, &link.canonicalize().unwrap()) + ObservedFile::open(temp.path(), &link, &link.canonicalize().unwrap()) .unwrap() .read() .unwrap(); @@ -548,13 +570,34 @@ mod tests { std::fs::write(&a, "actual original bytes").unwrap(); std::fs::write(&b, "different bytes").unwrap(); symlink(&a, &alias).unwrap(); - let reader = ObservedRead::open(temp.path(), &alias, &a.canonicalize().unwrap()).unwrap(); + let reader = ObservedFile::open(temp.path(), &alias, &a.canonicalize().unwrap()).unwrap(); std::fs::remove_file(&alias).unwrap(); symlink(&b, &alias).unwrap(); let (bytes, receipt) = reader.read().unwrap(); assert_eq!(bytes, b"actual original bytes"); assert!(receipt.verify().is_err()); - assert!(ObservedRead::open(temp.path(), &alias, &a.canonicalize().unwrap()).is_err()); + assert!(ObservedFile::open(temp.path(), &alias, &a.canonicalize().unwrap()).is_err()); + } + + #[test] + fn edit_receipt_requires_the_handle_that_wrote_the_observed_file() { + use std::io::Write; + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("target.txt"); + let other = temp.path().join("other.txt"); + std::fs::write(&path, "before").unwrap(); + let physical = path.canonicalize().unwrap(); + let observed = ObservedFile::open(temp.path(), &path, &physical).unwrap(); + let mut unrelated = std::fs::File::create(&other).unwrap(); + unrelated.write_all(b"after").unwrap(); + assert!(observed.after_write(&unrelated).is_err()); + + let observed = ObservedFile::open(temp.path(), &path, &physical).unwrap(); + let mut written = std::fs::File::create(&path).unwrap(); + written.write_all(b"after").unwrap(); + let receipt = observed.after_write(&written).unwrap(); + assert!(receipt.verify().is_ok()); + assert_eq!(receipt.evidence().path, physical); } fn root() -> (tempfile::TempDir, PathBuf) { diff --git a/crates/cyber-server/src/runtime/auto_override.rs b/crates/cyber-server/src/runtime/auto_override.rs index 4107b4dca..6f4595830 100644 --- a/crates/cyber-server/src/runtime/auto_override.rs +++ b/crates/cyber-server/src/runtime/auto_override.rs @@ -474,7 +474,7 @@ impl Inner { let nested = nested.pop().unwrap_or_default(); let mut state = handle.state.lock().await; let nested = run.current_nested(&state, nested); - let proof = run.read_proof(); + let proof = run.operation_proof(); let mut seen = super::drain::Reminded::of(&state); let (settled, loaded) = super::drain::settlement_with_reminders( &replay.id, @@ -487,7 +487,7 @@ impl Inner { .finalize_skill_output(&state.info.directory, output, reminder) }, ); - let settled = super::drain::with_read_proof(settled, proof); + let settled = super::drain::with_operation_proof(settled, &run.def.spec.name, proof); let session_id = state.info.id.clone(); let settled = self.with_call_metadata(&session_id, &replay.id, &state.info.directory, settled); @@ -504,7 +504,11 @@ impl Inner { self.instructions_loaded( handle, loaded.into_iter().collect(), - super::drain::NESTED_LOAD_REASON, + if run.def.spec.name == "edit" { + "nested_edit" + } else { + super::drain::NESTED_LOAD_REASON + }, ) .await; if let Some(reason) = &stop { diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 64b7ac92a..bdfec1682 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -1818,9 +1818,10 @@ impl Inner { let mut events = Vec::new(); let mut stops = Vec::new(); let mut loaded = Vec::new(); + let mut edited = Vec::new(); for (run, nested) in outcomes.into_iter().zip(nested) { let nested = run.current_nested(&state, nested); - let proof = run.read_proof(); + let proof = run.operation_proof(); let (settled, instructions) = settlement_with_reminders( &run.call_id, &run.def, @@ -1832,15 +1833,21 @@ impl Inner { .finalize_skill_output(&directory, output, reminder) }, ); - let settled = with_read_proof(settled, proof); + let settled = with_operation_proof(settled, &run.def.spec.name, proof); events.push(self.with_call_metadata(&session_id, &run.call_id, &directory, settled)); - loaded.extend(instructions); + if run.def.spec.name == "edit" { + edited.extend(instructions); + } else { + loaded.extend(instructions); + } stops.extend(self.take_call_hook_effects(&session_id, &run.call_id, &mut events)); } self.commit_locked(&mut state, events)?; drop(state); self.instructions_loaded(handle, loaded, NESTED_LOAD_REASON) .await; + self.instructions_loaded(handle, edited, "nested_edit") + .await; for reason in &stops { self.publish_hook_stop(&session_id, reason); } @@ -2034,7 +2041,7 @@ impl Inner { ) -> ToolRun { let call_id = invocation.call_id.clone(); let scope = ReminderScope::capture(&*handle.state.lock().await, &invocation.directory); - let read = scope.and_then(|scope| self.read_target(&def, &invocation, scope)); + let read = scope.and_then(|scope| self.file_target(&def, &invocation, scope)); if invocation.name == super::structured::CHILD_TOOL || invocation.name == super::structured::ROOT_TOOL { @@ -2060,12 +2067,12 @@ impl Inner { Ok(outcome) => outcome, Err(_) => { handle.location_uncertain.store(true, std::sync::atomic::Ordering::SeqCst); - super::host::ObservedExecution { outcome: ToolOutcome::Aborted, reads: Some(Vec::new()) } + super::host::ObservedExecution { outcome: ToolOutcome::Aborted, files: Some(Vec::new()) } } }, }; - let read = match outcome.reads { - None => read, + let read = match outcome.files { + None => read.filter(|_| def.spec.name == "read"), Some(mut receipts) if receipts.len() == 1 => read.and_then(|mut target| { let receipt = receipts.pop().unwrap(); if target.root.identity() != receipt.evidence().root_identity { @@ -2085,20 +2092,22 @@ impl Inner { } } - /// The file a built-in `read` call targets, for nested instructions. - fn read_target( + /// Capture the Location before an operation can suspend for authorization. + fn file_target( &self, def: &ToolDef, invocation: &Invocation, scope: ReminderScope, - ) -> Option { - if def.scope != super::host::ToolScope::Builtin || def.spec.name != "read" { + ) -> Option { + if def.scope != super::host::ToolScope::Builtin + || !matches!(def.spec.name.as_str(), "read" | "edit") + { return None; } let path = invocation.input.get("path")?.as_str()?; let settings = self.instruction_settings(&invocation.directory).ok()?; let location = std::path::PathBuf::from(&invocation.directory); - Some(ReadTarget { + Some(FileTarget { file: super::instructions::read_target(&location, settings.home.as_deref(), path), location: location.clone(), scope, @@ -2117,7 +2126,7 @@ impl Inner { pub(super) async fn nested_instructions( &self, handle: &Handle, - targets: Vec>, + targets: Vec>, ) -> Vec { let count = targets.len(); let empty = || (0..count).map(|_| Default::default()).collect(); @@ -2347,6 +2356,7 @@ fn settled( instruction_reminders: Vec::new(), rule_reminders: Vec::new(), rule_read: None, + rule_edit: None, structured_output: None, call_id: call_id.into(), status, @@ -2364,13 +2374,13 @@ pub(super) struct ToolRun { pub call_id: String, pub def: ToolDef, pub outcome: ToolOutcome, - pub read: Option, + pub read: Option, } impl ToolRun { - pub(super) fn read_proof(&self) -> Option { + pub(super) fn operation_proof(&self) -> Option { let target = self.read.as_ref()?; - Some(super::events::RuleRead { + Some(super::events::RuleOperation { evidence: target.receipt.as_ref()?.evidence().clone(), epoch: target.scope.epoch, location: target.scope.directory.clone(), @@ -2397,7 +2407,7 @@ impl ToolRun { } /// The target to load nested instructions for: only a read that succeeded. - pub(super) fn nested_target(&self) -> Option { + pub(super) fn nested_target(&self) -> Option { let succeeded = matches!( self.outcome, ToolOutcome::Ok(_) @@ -2439,16 +2449,16 @@ impl ReminderScope { /// A `read` call's Location, target and captured context; settings are resolved /// after successful execution rather than retaining pre-effect permission closures. #[derive(Clone)] -pub(super) struct ReadTarget { +pub(super) struct FileTarget { pub location: std::path::PathBuf, pub file: std::path::PathBuf, pub scope: ReminderScope, pub root: Arc, - pub receipt: Option>, + pub receipt: Option>, pub operation_key: String, pub attempt: u32, } -impl ReadTarget { +impl FileTarget { fn valid_native(&self) -> bool { self.root.verify().is_ok() && self @@ -2495,9 +2505,10 @@ impl ReadTarget { } } -pub(super) fn with_read_proof( +pub(super) fn with_operation_proof( mut settled: cyber_store::NewEvent, - proof: Option, + tool: &str, + proof: Option, ) -> cyber_store::NewEvent { if settled .data @@ -2506,8 +2517,12 @@ pub(super) fn with_read_proof( .is_some_and(|rules| !rules.is_empty()) && let Some(proof) = proof { - settled.data["rule_read"] = - serde_json::to_value(proof).expect("rule read metadata serializes"); + let field = if tool == "edit" { + "rule_edit" + } else { + "rule_read" + }; + settled.data[field] = serde_json::to_value(proof).expect("rule read metadata serializes"); } settled } @@ -2620,6 +2635,7 @@ pub(super) fn settlement_with_reminders( instruction_reminders: attached, rule_reminders: rule_bindings, rule_read: None, + rule_edit: None, structured_output: value, call_id: call_id.into(), status: if failed { @@ -2720,6 +2736,7 @@ pub(super) fn settlement( instruction_reminders: Vec::new(), rule_reminders: Vec::new(), rule_read: None, + rule_edit: None, structured_output: None, call_id: call_id.into(), status: CallStatus::Ok, @@ -2754,6 +2771,7 @@ pub(super) fn settlement( instruction_reminders: Vec::new(), rule_reminders: Vec::new(), rule_read: None, + rule_edit: None, structured_output: Some(value), call_id: call_id.into(), status: CallStatus::Ok, @@ -2779,6 +2797,7 @@ pub(super) fn settlement( instruction_reminders: Vec::new(), rule_reminders: Vec::new(), rule_read: None, + rule_edit: None, structured_output: value, call_id: call_id.into(), status: if failed { @@ -2922,7 +2941,7 @@ mod skill_reminder_tests { call_id: "read".into(), def: read_def(), outcome: ToolOutcome::Ok("file".into()), - read: Some(ReadTarget { + read: Some(FileTarget { location: "/repo".into(), file: "/repo/web/app.ts".into(), scope, @@ -2951,7 +2970,7 @@ mod skill_reminder_tests { if state.epoch.is_some() { let scope = ReminderScope::capture(&state, &state.info.directory).unwrap(); let fresh = ToolRun { - read: Some(ReadTarget { + read: Some(FileTarget { location: state.info.directory.clone().into(), file: "/repo/web/app.ts".into(), scope, @@ -3170,7 +3189,7 @@ mod skill_reminder_tests { #[test] fn only_successful_reads_load_nested_instructions() { let root_dir = tempfile::tempdir().unwrap(); - let read = ReadTarget { + let read = FileTarget { location: "/repo".into(), file: "/repo/web/app.ts".into(), scope: ReminderScope { @@ -3251,7 +3270,7 @@ mod skill_reminder_tests { call_id: "call".into(), def: read_def(), outcome: external(false), - read: Some(ReadTarget { + read: Some(FileTarget { location: "/repo".into(), file: "/repo/a.md".into(), scope: ReminderScope { diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 3341be836..70cce9c77 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -367,23 +367,24 @@ pub struct ToolDispatched { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RuleRead { - pub evidence: cyber_core::rules::ReadEvidence, +pub struct RuleOperation { + pub evidence: cyber_core::rules::FileEvidence, pub epoch: u32, pub location: String, pub operation_key: String, pub attempt: u32, } -impl RuleRead { +impl RuleOperation { pub fn matches( &self, binding: &super::instructions::RuleBinding, epoch: u32, + tool: &str, info: &super::model::SessionInfo, call: &super::model::CallState, ) -> bool { call.status == super::model::CallStatus::Dispatched - && call.name.eq_ignore_ascii_case("read") + && call.name.eq_ignore_ascii_case(tool) && self.epoch == epoch && self.location == info.directory && self.attempt != 0 @@ -409,7 +410,9 @@ pub struct ToolSettled { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub rule_reminders: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] - pub rule_read: Option, + pub rule_read: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rule_edit: Option, #[serde( default, skip_serializing_if = "Option::is_none", diff --git a/crates/cyber-server/src/runtime/host.rs b/crates/cyber-server/src/runtime/host.rs index a25e5a838..4a0eb3234 100644 --- a/crates/cyber-server/src/runtime/host.rs +++ b/crates/cyber-server/src/runtime/host.rs @@ -488,12 +488,12 @@ pub struct SkillSuggestion { pub description: String, } -/// An execution with actual file reads, when the host instruments them. `None` -/// preserves legacy ordinary instruction loading; `Some([])` proves no eligible read. +/// An execution with actual file operations, when the host instruments them. `None` +/// preserves legacy ordinary instruction loading; `Some([])` reports no eligible operation. #[derive(Debug)] pub struct ObservedExecution { pub outcome: ToolOutcome, - pub reads: Option>>, + pub files: Option>>, } /// How a tool run ended. @@ -1084,7 +1084,7 @@ pub trait ToolHost: Send + Sync { /// Run a call. Implementations stop within 2 s once `cancel` fires. fn execute(&self, call: Invocation, cancel: CancellationToken) -> BoxFuture<'_, ToolOutcome>; - /// Run with native read observations when the host can supply them. The + /// Run with native file observations when the host can supply them. The /// default has no operation receipt and cannot activate scoped rules. fn execute_observed( &self, @@ -1094,7 +1094,7 @@ pub trait ToolHost: Send + Sync { Box::pin(async move { ObservedExecution { outcome: self.execute(invocation, cancel).await, - reads: None, + files: None, } }) } diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 553ddd93c..7828e9330 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -2318,6 +2318,7 @@ impl Runtime { instruction_reminders: Vec::new(), rule_reminders: Vec::new(), rule_read: None, + rule_edit: None, structured_output: None, call_id: call_id.into(), status, diff --git a/crates/cyber-server/src/runtime/model.rs b/crates/cyber-server/src/runtime/model.rs index 6cc89b1ed..05f2c6ac1 100644 --- a/crates/cyber-server/src/runtime/model.rs +++ b/crates/cyber-server/src/runtime/model.rs @@ -1512,13 +1512,31 @@ impl SessionState { epoch.reminded_rules.extend( s.rule_reminders .iter() - .filter(|binding| match (&binding.native_root, &s.rule_read) { - (None, None) => true, // historical, path-only binding - (Some(_), Some(read)) => { - !self.epoch_stale - && read.matches(binding, epoch.number, &self.info, call) + .filter(|binding| { + match (&binding.native_root, &s.rule_read, &s.rule_edit) { + (None, None, None) => true, + (Some(_), Some(read), None) => { + !self.epoch_stale + && read.matches( + binding, + epoch.number, + "read", + &self.info, + call, + ) + } + (Some(_), None, Some(edit)) => { + !self.epoch_stale + && edit.matches( + binding, + epoch.number, + "edit", + &self.info, + call, + ) + } + _ => false, } - _ => false, }) .map(super::instructions::RuleBinding::identity), ); diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index 096c9b263..f80d09016 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1611,7 +1611,7 @@ impl ToolHost for BuiltinHost { cancel: CancellationToken, ) -> BoxFuture<'_, cyber_server::runtime::ObservedExecution> { Box::pin(async move { - let read_receipt = ReadReceiptSlot::default(); + let read_receipt = FileReceiptSlot::default(); let outcome = self.execute_inner(inv, cancel, read_receipt.clone()).await; let read = read_receipt .lock() @@ -1619,7 +1619,7 @@ impl ToolHost for BuiltinHost { .take(); cyber_server::runtime::ObservedExecution { outcome, - reads: Some(read.into_iter().collect()), + files: Some(read.into_iter().collect()), } }) } @@ -1959,7 +1959,7 @@ impl BuiltinHost { hook_decision: Option, cancel: CancellationToken, denial: DenialHooks, - read_receipt: ReadReceiptSlot, + read_receipt: FileReceiptSlot, ) -> Result, String> { let policy = self.policy(inv).await?; if let Some(reason) = policy.skill_tool_denial(&inv.name) { @@ -2056,10 +2056,10 @@ impl BuiltinHost { } /// Everything a tool needs for one call. -pub(crate) type ReadReceiptSlot = Arc>>>; +pub(crate) type FileReceiptSlot = Arc>>>; pub(crate) struct Ctx<'a> { - pub read_receipt: ReadReceiptSlot, + pub read_receipt: FileReceiptSlot, pub skill_paths: Mutex>, pub loaded_skill: Mutex>, /// Set by a tool whose result comes from outside the workspace; the projection @@ -2799,7 +2799,7 @@ impl BuiltinHost { &self, inv: Invocation, cancel: CancellationToken, - read_receipt: ReadReceiptSlot, + read_receipt: FileReceiptSlot, ) -> BoxFuture<'_, ToolOutcome> { Box::pin(async move { if inv.name.starts_with("mcp__") { diff --git a/crates/cyber-tools/src/tools/fs.rs b/crates/cyber-tools/src/tools/fs.rs index 0280c1631..265b85985 100644 --- a/crates/cyber-tools/src/tools/fs.rs +++ b/crates/cyber-tools/src/tools/fs.rs @@ -113,7 +113,7 @@ fn observed_reader( ctx: &Ctx<'_>, path: &Path, authorized: &Path, -) -> Option { +) -> Option { let location = ctx.location.canonicalize().ok()?; let physical_resource = authorized.strip_prefix(&location).map_or_else( |_| ctx.resource(authorized), @@ -134,7 +134,7 @@ fn observed_reader( return None; } } - cyber_core::rules::ObservedRead::open(&ctx.location, path, authorized).ok() + cyber_core::rules::ObservedFile::open(&ctx.location, path, authorized).ok() } /// A page of the file redacted as a whole first, so a secret that the page boundary @@ -424,6 +424,8 @@ pub(crate) async fn guarded_write( &String::from_utf8_lossy(after), )?; ctx.check_external_writes(&[path.to_path_buf()]).await?; + // Capture the physical spelling before permission can suspend the call. + let authorized = path.canonicalize().ok(); authorize_edit(ctx, path, diff).await?; let origin = super::intelligence::capture(ctx, path); let workspace = cyber_server::workspace::mutation_lock(&ctx.location) @@ -442,7 +444,7 @@ pub(crate) async fn guarded_write( .await .map_err(|e| failed(format!("Could not create {}: {e}", parent.display())))?; } - tokio::fs::write(path, after) + write_observed_edit(ctx, path, after, authorized.as_deref()) .await .map_err(|e| failed(format!("Could not write {}: {e}", path.display())))?; ctx.host.mark_read(&ctx.inv.session_id, path); @@ -452,6 +454,47 @@ pub(crate) async fn guarded_write( Ok(super::intelligence::feedback(ctx, path, after, origin).await) } +async fn write_observed_edit( + ctx: &Ctx<'_>, + path: &Path, + bytes: &[u8], + authorized: Option<&Path>, +) -> std::io::Result<()> { + // An edit approval for an alias does not attest a separately named resource. + let observation = authorized + .filter(|_| ctx.inv.name == "edit") + .filter(|target| { + ctx.location + .canonicalize() + .ok() + .and_then(|location| { + target + .strip_prefix(location) + .ok() + .map(|relative| ctx.location.join(relative)) + }) + .as_deref() + == Some(path) + }) + .and_then(|target| cyber_core::rules::ObservedFile::open(&ctx.location, path, target).ok()); + let Some(observation) = observation else { + return tokio::fs::write(path, bytes).await; + }; + use tokio::io::AsyncWriteExt; + let mut file = tokio::fs::File::create(path).await?; + file.write_all(bytes).await?; + file.flush().await?; + let file = file.into_std().await; + let receipt = tokio::task::spawn_blocking(move || observation.after_write(&file)).await; + if let Ok(Ok(receipt)) = receipt { + *ctx.read_receipt + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(std::sync::Arc::new(receipt)); + } + Ok(()) +} + pub(crate) async fn authorize_edit( ctx: &Ctx<'_>, path: &Path, diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 359ab17e1..71dd0d3e8 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native read and dispatched attempt. Finish edit/patch/partial-effect delivery, exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads and edits use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native read or edit and dispatched attempt. Finish patch and partial-effect delivery, exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 423233c67..51ad0a25e 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1221,9 +1221,9 @@ Project rule bytes are included in the existing checkout trust digest. Review wi Definition reads use the fenced native reader: linked files/directories, hard links and special files are refused. The reader caps a definition at 1 MiB; frontmatter is limited to 16 KiB, with up to 64 relative patterns of 256 bytes each. A project inventory covers up to 128 definitions and 4 MiB, with bounded directory enumeration. These review/discovery bounds are separate from the smaller rendered instruction budgets. Unknown inventories grant no rule authority. `cyber doctor` reports held, refused and truncated baseline rules. -Scoped rules activate after a successful built-in file read with a native receipt. The tool matches the physical file it actually opened after authorization and any hook rewrite. A directory listing, unsupported observation, or changed root/target binding does not consume delivery. The tool's own result is preserved. Ordinary instruction behavior remains available to legacy hosts without native receipts; those hosts cannot activate path-scoped rules. +Scoped rules activate after a successful built-in file read or edit with a native receipt. Rule matching uses the physical file from the completed operation after authorization and any hook rewrite. Edit receipts require the handle that completed the write to match the observed file. A directory listing, unsupported observation, or changed root/target binding does not consume delivery. The tool's own result is preserved. Ordinary instruction behavior remains available to legacy hosts without native receipts; those hosts cannot activate path-scoped rules. -The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Edit/patch and partial-settlement delivery, exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. +The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Patch and partial-settlement delivery, exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 6b0284e93..09e65b567 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -28,7 +28,7 @@ Definition discovery bounds are 128 project files, 4096 examined entries per rul Nested read discovery resolves instruction settings after a successful effect. Its captured Session Location and Context Epoch must still be active before discovery and again under the settlement lock. An absent/stale/replaced Epoch or changed Location suppresses the reminder without recording delivery; normal group settlement and exact `/approve` replay share this check. The captured read also retains the native directory identities from the volume root through the physical Location, verifies the project-root boundary, and rechecks those bindings before/after discovery and under settlement. Unix retains directory handles to prevent inode reuse; Windows releases handles after capture and verifies full native IDs, so the observation does not freeze editor/worktree renames. Ordinary file content edits do not invalidate the root. Replacement/removal, changed ancestors, linked substitution, a retargeted declared Location alias or a changed project boundary withhold the reminder. These point-in-time context/root checks do not prove which file handle performed the effect: actual per-file operation receipts (including partial writes), exact configuration-generation fencing through commit and revocation withdrawal remain required. -Built-in reads now read through an observed file handle after authorization and any input rewrite, and hand the resulting receipt to the runtime through the host execution boundary. Workspace read targets preserve the normal size and hard-link behavior; rule-definition and memory-storage source readers keep their stricter checks. The receipt retains actual native file/ancestor identities, a physical path and the captured project root. Unsupported observations keep the read result but grant no scoped-rule delivery. Rechecking a receipt reads metadata, not file contents. The ordinary legacy host path cannot activate scoped definitions. Successful labelled delivery stores `rule_read` atomically with `rule_reminders`: actual target/root IDs, captured Location/Epoch, operation key and dispatched attempt. Version-2 rule IDs incorporate the native root; old IDs and legacy records replay unchanged. The recorded proof describes an earlier operation and never grants new filesystem authority. Replay checks the dispatch and context before folding new identities. Actual edit/patch receipts, positive partial effects, exact configuration generation and revocation/restart acceptance remain open. +Built-in reads now read through an observed file handle after authorization and any input rewrite, and hand the resulting receipt to the runtime through the host execution boundary. Workspace read targets preserve the normal size and hard-link behavior; rule-definition and memory-storage source readers keep their stricter checks. The receipt retains actual native file/ancestor identities, a physical path and the captured project root. Unsupported observations keep the read result but grant no scoped-rule delivery. Rechecking a receipt reads metadata, not file contents. The ordinary legacy host path cannot activate scoped definitions. Successful labelled delivery stores `rule_read` atomically with `rule_reminders`: actual target/root IDs, captured Location/Epoch, operation key and dispatched attempt. Version-2 rule IDs incorporate the native root; old IDs and legacy records replay unchanged. The recorded proof describes an earlier operation and never grants new filesystem authority. Replay checks the dispatch and context before folding new identities. An existing-file edit compares the completed writer handle with the observed native file and records `rule_edit` under the same dispatch and Context Epoch checks. A failed read leaves its scoped rule eligible for that edit. Patch receipts, positive partial effects, exact configuration generation and revocation/restart acceptance remain open. Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 28bccd9b0..9ed4b2bd7 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1797,3 +1797,18 @@ reader's import tests are included in the platform workflow. Windows, whole workspace, SDK runtime and current-source native matrix acceptance remain unrun locally for this increment. Checkpoint review found no remaining findings in the read changes after the fixes above; full-goal review remains open. + +Existing-file edit continuation. The guarded edit captures the authorized +physical path and verifies the handle that completed the write against the +observed file. The settlement stores the edit receipt under `rule_edit` with the +native root and file identities, dispatched attempt, Location and Context Epoch. +Replay checks those fields against the edit dispatch. The App regression covers +a denied read followed by a successful edit in the same Epoch. The Core test +rejects a writer handle for an unrelated file. An edit through a separately named +alias can still produce its normal tool result without a scoped rule receipt. +Multi-file patch, creation/removal receipts, partial patch settlements, exact +configuration generation and revocation/restart acceptance remain open. + +Local edit verification passed 12 filtered Core tests, 17 App instruction tests, +and all-target Clippy for Core, Tools, Server and App. Formatting, SDK consistency +and strict OpenSpec checks passed. Windows and platform acceptance remain pending. From aeb12a63c95f2041d53db1694e82b2094faa4812 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 18:52:16 -0400 Subject: [PATCH 81/87] Deliver scoped rules for completed patch effects --- .github/workflows/non-git-snapshot-proof.yml | 1 + crates/cyber-app/tests/instructions.rs | 118 +++++++++++ crates/cyber-core/src/import/mod.rs | 4 +- crates/cyber-core/src/import/snapshot.rs | 89 ++++++++- crates/cyber-core/src/rules.rs | 175 ++++++++++++++++- .../cyber-server/src/runtime/auto_override.rs | 8 +- crates/cyber-server/src/runtime/drain.rs | 184 ++++++++++++------ crates/cyber-server/src/runtime/events.rs | 12 ++ .../cyber-server/src/runtime/instructions.rs | 79 +++++--- crates/cyber-server/src/runtime/mod.rs | 1 + crates/cyber-server/src/runtime/model.rs | 84 ++++---- crates/cyber-tools/src/host.rs | 15 +- crates/cyber-tools/src/tools/fs.rs | 114 ++++++++--- crates/cyber-tools/src/tools/patch.rs | 32 ++- docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 4 +- .../complete-daily-coding-surfaces/design.md | 2 +- .../specs/system-context/spec.md | 9 + .../complete-daily-coding-surfaces/tasks.md | 23 +++ 19 files changed, 772 insertions(+), 184 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index f7c18365a..665783132 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -36,6 +36,7 @@ on: - crates/cyber-core/tests/config.rs - crates/cyber-tools/src/host.rs - crates/cyber-tools/src/tools/fs.rs + - crates/cyber-tools/src/tools/patch.rs - crates/cyber-server/src/runtime/auto_override.rs - crates/cyber-tools/tests/instruction_config.rs - crates/cyber-tools/tests/path_skills.rs diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index 7e5e768e0..702f46f9e 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -28,6 +28,20 @@ fn reply(body: &Value) -> Value { return json!({"content": "done"}); } let text = last.to_string(); + if text.contains("patch the app") { + let mut patch = + "*** Begin Patch\n*** Update File: web/a.txt\n@@\n-before-a\n+after-a\n".to_string(); + if text.contains("partially") { + patch.push_str("*** Update File: web/a.txt\n@@\n-before-a\n+second-a\n"); + } + patch.push_str("*** Update File: db/b.txt\n@@\n-before-b\n+after-b\n"); + patch.push_str( + "*** Add File: assets/new/c.txt\n+created-c\n*** Delete File: old/d.txt\n*** End Patch", + ); + let arguments = json!({"patch":patch}).to_string(); + return json!({"tool_calls":[{"index":0,"id":"call_patch","type":"function", + "function":{"name":"apply_patch","arguments":arguments}}]}); + } if text.contains("edit the app") { let arguments = json!({"path":"web/src/app.ts","old_string":"const version = 1","new_string":"const version = 2"}).to_string(); return json!({"tool_calls":[{"index":0,"id":"call_edit","type":"function", @@ -385,6 +399,10 @@ impl Fixture { /// Start with `extra` top-level keys added to the global configuration. async fn start_with(extra: Value) -> Self { + Self::start_with_model(extra, false).await + } + + async fn start_with_model(extra: Value, prefers_patch: bool) -> Self { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); let paths = Paths { @@ -397,6 +415,10 @@ impl Fixture { paths.ensure().unwrap(); let model = ScriptedModel::start(reply); let mut global = model.config(); + if prefers_patch { + global["providers"]["test"]["models"]["main"] = + json!({"capabilities":{"prefers_apply_patch":true}}); + } for (key, value) in extra.as_object().unwrap() { global[key] = value.clone(); } @@ -948,3 +970,99 @@ async fn denied_read_leaves_rule_available_for_a_successful_edit() { assert_eq!(replay.epoch.unwrap().reminded_rules, epoch.reminded_rules); f.app.runtime.shutdown().await; } + +#[tokio::test] +async fn patch_rules_follow_completed_files_even_after_a_later_failure() { + use cyber_server::runtime::CallStatus; + for partial in [true, false] { + let f = Fixture::start_with_model(json!({}), true).await; + f.write("web/a.txt", "before-a\n"); + f.write("db/b.txt", "before-b\n"); + f.write( + ".cyber/rules/web.md", + "---\npaths: ['web/**']\n---\nPATCH_WEB_RULE", + ); + f.write( + ".cyber/rules/db.md", + "---\npaths: ['db/**']\n---\nPATCH_DB_RULE", + ); + f.write( + ".cyber/rules/assets.md", + "---\npaths: ['assets/**']\n---\nPATCH_ASSET_RULE", + ); + f.write("old/d.txt", "delete me\n"); + f.write( + ".cyber/rules/old.md", + "---\npaths: ['old/**']\n---\nPATCH_REMOVED_RULE", + ); + f.approve(&f.repo); + let id = f.session(&f.repo).await; + let prompt = if partial { + "patch the app partially" + } else { + "patch the app" + }; + f.prompt(&id, &f.repo, prompt, 2).await; + + let result = last_tool_result(&f.last_request()); + assert!(result.contains("PATCH_WEB_RULE"), "{result}"); + assert_eq!(result.contains("PATCH_DB_RULE"), !partial, "{result}"); + assert_eq!(result.contains("PATCH_ASSET_RULE"), !partial, "{result}"); + assert_eq!(result.contains("PATCH_REMOVED_RULE"), !partial, "{result}"); + assert_eq!(f.repo.join("old/d.txt").exists(), partial); + assert_eq!( + std::fs::read_to_string(f.repo.join("assets/new/c.txt")) + .ok() + .as_deref(), + (!partial).then_some("created-c\n") + ); + assert_eq!( + std::fs::read_to_string(f.repo.join("web/a.txt")).unwrap(), + "after-a\n" + ); + assert_eq!( + std::fs::read_to_string(f.repo.join("db/b.txt")).unwrap(), + if partial { "before-b\n" } else { "after-b\n" } + ); + let state = f.app.runtime.state(&id).await.unwrap(); + assert_eq!( + state.calls["call_patch"].status, + if partial { + CallStatus::Error + } else { + CallStatus::Ok + } + ); + let ids = state.epoch.unwrap().reminded_rules; + assert_eq!(ids.len(), if partial { 1 } else { 4 }); + let events = f.app.store.read_events(&id, -1, 500).unwrap().events; + let settled = events + .iter() + .find(|event| event.data["rule_patch"].is_array()) + .unwrap(); + let proof = settled.data["rule_patch"].as_array().unwrap(); + assert_eq!(proof.len(), ids.len()); + if !partial { + assert!(proof[3]["evidence"]["removed_parent_identity"].is_string()); + assert_eq!( + proof[3]["evidence"]["path"], + f.repo.join("old/d.txt").to_str().unwrap() + ); + } + assert_eq!( + proof[0]["evidence"]["path"], + f.repo.join("web/a.txt").to_str().unwrap() + ); + let mut wrong_target = events.clone(); + let event = wrong_target + .iter_mut() + .find(|event| event.id == settled.id) + .unwrap(); + event.data["rule_patch"][0]["evidence"]["path"] = json!(f.repo.join("web/wrong.txt")); + let replay = cyber_server::runtime::SessionState::replay(&wrong_target).unwrap(); + assert_eq!(replay.epoch.unwrap().reminded_rules.len(), ids.len() - 1); + let replay = cyber_server::runtime::SessionState::replay(&events).unwrap(); + assert_eq!(replay.epoch.unwrap().reminded_rules, ids); + f.app.runtime.shutdown().await; + } +} diff --git a/crates/cyber-core/src/import/mod.rs b/crates/cyber-core/src/import/mod.rs index 3078a653a..4b5e443e1 100644 --- a/crates/cyber-core/src/import/mod.rs +++ b/crates/cyber-core/src/import/mod.rs @@ -46,8 +46,8 @@ pub use preview::{ pub use provenance::SourceReference; pub use snapshot::SourceSnapshot; pub(crate) use snapshot::{ - DirectorySnapshot, FileHandleReceipt, ObservedFileHandle, command_directory, - verify_command_directory, + DirectorySnapshot, FileHandleReceipt, ObservedFileHandle, ObservedRemoval, RemovalReceipt, + command_directory, verify_command_directory, }; mod opencode_providers; diff --git a/crates/cyber-core/src/import/snapshot.rs b/crates/cyber-core/src/import/snapshot.rs index 3aa43c1c0..01b5d5cee 100644 --- a/crates/cyber-core/src/import/snapshot.rs +++ b/crates/cyber-core/src/import/snapshot.rs @@ -144,6 +144,12 @@ pub(crate) struct DirectorySnapshot { impl DirectorySnapshot { pub(crate) fn capture(path: &Path) -> Result { let (directories, bindings) = directory_chain(path)?; + let snapshot = Self::from_chain(path.into(), directories, bindings); + snapshot.verify()?; + Ok(snapshot) + } + + fn from_chain(path: PathBuf, directories: Vec, bindings: Vec) -> Self { // NTFS directory handles would freeze ordinary editor/worktree renames. // Full Windows file IDs retain reuse protection without open handles. #[cfg(windows)] @@ -153,13 +159,11 @@ impl DirectorySnapshot { }; #[cfg(not(windows))] let retained = Some(directories); - let snapshot = Self { - path: path.into(), + Self { + path, bindings, _retained: retained, - }; - snapshot.verify()?; - Ok(snapshot) + } } pub(crate) fn verify(&self) -> Result<(), DiscoveryError> { let (_, bindings) = directory_chain(&self.path)?; @@ -273,6 +277,81 @@ impl FileHandleReceipt { } } +/// A removal carried out relative to the pinned physical parent directory. +pub(crate) struct ObservedRemoval { + path: PathBuf, + directories: Vec, + bindings: Vec, +} + +pub(crate) struct RemovalReceipt { + parent: DirectorySnapshot, + path: PathBuf, +} + +impl ObservedRemoval { + pub(crate) fn open(path: &Path) -> Result { + let parent = path + .parent() + .ok_or_else(|| error(path, "removal has no parent"))?; + let name = path + .file_name() + .ok_or_else(|| error(path, "removal has no filename"))?; + let (directories, bindings) = directory_chain(parent)?; + let metadata = directories + .last() + .unwrap() + .symlink_metadata(name) + .map_err(|_| error(path, "removal target is unavailable"))?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(error(path, "removal target is not a regular file")); + } + Ok(Self { + path: path.into(), + directories, + bindings, + }) + } + + pub(crate) fn remove(self) -> std::io::Result { + let (_, current) = directory_chain(self.path.parent().unwrap()) + .map_err(|error| std::io::Error::other(error.reason))?; + if current != self.bindings { + return Err(std::io::Error::other("authorized removal parent changed")); + } + self.directories + .last() + .unwrap() + .remove_file(self.path.file_name().unwrap())?; + let parent = DirectorySnapshot::from_chain( + self.path.parent().unwrap().into(), + self.directories, + self.bindings, + ); + Ok(RemovalReceipt { + parent, + path: self.path, + }) + } +} + +impl RemovalReceipt { + pub(crate) fn verify(&self) -> Result<(), DiscoveryError> { + self.parent.verify()?; + match std::fs::symlink_metadata(&self.path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => self.parent.verify(), + _ => Err(error( + &self.path, + "removed target no longer has a verified absence", + )), + } + } + + pub(crate) fn parent_identity(&self) -> String { + fingerprint(&self.parent.bindings) + } +} + /// Internal read-time command discovery keeps every directory binding live during enumeration. pub(crate) fn command_directory(path: &Path) -> Result, DiscoveryError> { directory_chain(path).map(|(directories, _)| directories) diff --git a/crates/cyber-core/src/rules.rs b/crates/cyber-core/src/rules.rs index 11acb8652..27cd65c0b 100644 --- a/crates/cyber-core/src/rules.rs +++ b/crates/cyber-core/src/rules.rs @@ -6,6 +6,9 @@ use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +/// Bound retained native file handles and rule discovery per tool settlement. +pub const MAX_OPERATION_RECEIPTS: usize = 32; + pub const MAX_FILES: usize = 128; pub const MAX_TOTAL_BYTES: usize = 4 * 1024 * 1024; const MAX_ENTRIES: usize = 4096; @@ -59,14 +62,22 @@ impl std::fmt::Debug for RootBinding { } } -/// Metadata describing a successful read; these hashes are observations, never +/// Metadata describing a completed file effect. These hashes are observations, never /// authority to perform another operation or reopen a source after restart. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct FileEvidence { pub root: PathBuf, pub path: PathBuf, pub root_identity: String, - pub file_identity: String, + #[serde(flatten)] + pub target: OperationTarget, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +pub enum OperationTarget { + File { file_identity: String }, + Removed { removed_parent_identity: String }, } pub struct ObservedFile { declared: PathBuf, @@ -74,12 +85,120 @@ pub struct ObservedFile { root: RootBinding, reader: crate::import::ObservedFileHandle, } +/// A new file's authorized physical path and parent binding before creation. +pub struct PendingCreation { + declared: PathBuf, + physical: PathBuf, + root: RootBinding, + parent: crate::import::DirectorySnapshot, +} + +impl PendingCreation { + pub fn capture(location: &Path, path: &Path, authorized: &Path) -> Result { + let root = RootBinding::capture(location)?; + let declared = std::path::absolute(path).map_err(|_| "creation path is invalid")?; + let directory = declared + .parent() + .ok_or("creation has no parent")? + .canonicalize() + .map_err(|_| "creation parent is unavailable")?; + let physical = directory.join(declared.file_name().ok_or("creation has no filename")?); + if physical != authorized + || !physical.starts_with(root.root()) + || physical.to_str().is_none() + || root.root().to_str().is_none() + { + return Err("creation target differs from the authorized workspace path".into()); + } + let parent = crate::import::DirectorySnapshot::capture(&directory) + .map_err(|error| error.reason.to_owned())?; + root.verify()?; + Ok(Self { + declared, + physical, + root, + parent, + }) + } + + /// The caller completed an exclusive creation through this handle. + pub fn after_write(self, file: &std::fs::File) -> Result { + let native = crate::import::ObservedFileHandle::open(&self.physical) + .and_then(|observed| observed.after_write(file)) + .map_err(|error| error.reason.to_owned())?; + self.parent + .verify() + .map_err(|error| error.reason.to_owned())?; + let receipt = FileReceipt::from_native(self.declared, self.physical, self.root, native); + receipt.verify()?; + Ok(receipt) + } +} + pub struct FileReceipt { declared: PathBuf, root: RootBinding, - native: crate::import::FileHandleReceipt, + native: NativeReceipt, evidence: FileEvidence, } +enum NativeReceipt { + File(crate::import::FileHandleReceipt), + Removed(crate::import::RemovalReceipt), +} + +pub struct PendingRemoval { + declared: PathBuf, + physical: PathBuf, + root: RootBinding, + operation: crate::import::ObservedRemoval, +} + +impl PendingRemoval { + pub fn capture(location: &Path, path: &Path, authorized: &Path) -> Result { + let root = RootBinding::capture(location)?; + let declared = std::path::absolute(path).map_err(|_| "removal path is invalid")?; + let physical = declared + .canonicalize() + .map_err(|_| "removal target is unavailable")?; + if physical != authorized + || !physical.starts_with(root.root()) + || physical.to_str().is_none() + || root.root().to_str().is_none() + { + return Err("removal target differs from the authorized workspace path".into()); + } + let operation = crate::import::ObservedRemoval::open(&physical) + .map_err(|error| error.reason.to_owned())?; + root.verify()?; + Ok(Self { + declared, + physical, + root, + operation, + }) + } + + pub fn remove(self) -> std::io::Result> { + self.root.verify().map_err(std::io::Error::other)?; + let removed = self.operation.remove()?; + let evidence = FileEvidence { + root: self.root.root().into(), + path: self.physical, + root_identity: self.root.identity(), + target: OperationTarget::Removed { + removed_parent_identity: removed.parent_identity(), + }, + }; + let receipt = FileReceipt { + declared: self.declared, + root: self.root, + native: NativeReceipt::Removed(removed), + evidence, + }; + Ok(receipt.verify().is_ok().then_some(receipt)) + } +} + impl RootBinding { pub fn identity(&self) -> String { self.native @@ -145,12 +264,14 @@ impl FileReceipt { root: root.root().into(), path: physical, root_identity: root.identity(), - file_identity: native.identity(), + target: OperationTarget::File { + file_identity: native.identity(), + }, }; Self { declared, root, - native, + native: NativeReceipt::File(native), evidence, } } @@ -160,10 +281,26 @@ impl FileReceipt { } pub fn verify(&self) -> Result<(), String> { self.root.verify()?; - if self.declared.canonicalize().ok().as_ref() != Some(&self.evidence.path) { - return Err("declared read target changed since the operation".into()); + match &self.native { + NativeReceipt::File(native) => { + if self.declared.canonicalize().ok().as_ref() != Some(&self.evidence.path) { + return Err("declared operation target changed".into()); + } + native.verify().map_err(|error| error.reason.to_owned()) + } + NativeReceipt::Removed(native) => { + let parent = self + .declared + .parent() + .and_then(|path| path.canonicalize().ok()); + if parent.as_deref() != self.evidence.path.parent() + || self.declared.file_name() != self.evidence.path.file_name() + { + return Err("declared removal parent changed".into()); + } + native.verify().map_err(|error| error.reason.to_owned()) + } } - self.native.verify().map_err(|e| e.reason.to_owned()) } } impl std::fmt::Debug for FileReceipt { @@ -600,6 +737,28 @@ mod tests { assert_eq!(receipt.evidence().path, physical); } + #[cfg(unix)] + #[test] + fn pending_removal_refuses_a_replaced_parent_before_unlinking() { + let temp = tempfile::tempdir().unwrap(); + let parent = temp.path().join("parent"); + std::fs::create_dir(&parent).unwrap(); + let path = parent.join("target.txt"); + std::fs::write(&path, "original").unwrap(); + let observed = + PendingRemoval::capture(temp.path(), &path, &path.canonicalize().unwrap()).unwrap(); + let retained = temp.path().join("retained"); + std::fs::rename(&parent, &retained).unwrap(); + std::fs::create_dir(&parent).unwrap(); + std::fs::write(&path, "replacement").unwrap(); + assert!(observed.remove().is_err()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "replacement"); + assert_eq!( + std::fs::read_to_string(retained.join("target.txt")).unwrap(), + "original" + ); + } + fn root() -> (tempfile::TempDir, PathBuf) { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); diff --git a/crates/cyber-server/src/runtime/auto_override.rs b/crates/cyber-server/src/runtime/auto_override.rs index 6f4595830..0e184f58f 100644 --- a/crates/cyber-server/src/runtime/auto_override.rs +++ b/crates/cyber-server/src/runtime/auto_override.rs @@ -504,10 +504,10 @@ impl Inner { self.instructions_loaded( handle, loaded.into_iter().collect(), - if run.def.spec.name == "edit" { - "nested_edit" - } else { - super::drain::NESTED_LOAD_REASON + match run.def.spec.name.as_str() { + "edit" => "nested_edit", + "apply_patch" => "nested_patch", + _ => super::drain::NESTED_LOAD_REASON, }, ) .await; diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index bdfec1682..ab71166e5 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -1819,6 +1819,7 @@ impl Inner { let mut stops = Vec::new(); let mut loaded = Vec::new(); let mut edited = Vec::new(); + let mut patched = Vec::new(); for (run, nested) in outcomes.into_iter().zip(nested) { let nested = run.current_nested(&state, nested); let proof = run.operation_proof(); @@ -1835,10 +1836,10 @@ impl Inner { ); let settled = with_operation_proof(settled, &run.def.spec.name, proof); events.push(self.with_call_metadata(&session_id, &run.call_id, &directory, settled)); - if run.def.spec.name == "edit" { - edited.extend(instructions); - } else { - loaded.extend(instructions); + match run.def.spec.name.as_str() { + "edit" => edited.extend(instructions), + "apply_patch" => patched.extend(instructions), + _ => loaded.extend(instructions), } stops.extend(self.take_call_hook_effects(&session_id, &run.call_id, &mut events)); } @@ -1848,6 +1849,8 @@ impl Inner { .await; self.instructions_loaded(handle, edited, "nested_edit") .await; + self.instructions_loaded(handle, patched, "nested_patch") + .await; for reason in &stops { self.publish_hook_stop(&session_id, reason); } @@ -2073,15 +2076,22 @@ impl Inner { }; let read = match outcome.files { None => read.filter(|_| def.spec.name == "read"), - Some(mut receipts) if receipts.len() == 1 => read.and_then(|mut target| { - let receipt = receipts.pop().unwrap(); - if target.root.identity() != receipt.evidence().root_identity { - return None; - } - target.file = receipt.evidence().path.clone(); - target.receipt = Some(receipt); - Some(target) - }), + Some(receipts) + if !receipts.is_empty() + && receipts.len() <= cyber_core::rules::MAX_OPERATION_RECEIPTS + && (def.spec.name == "apply_patch" || receipts.len() == 1) => + { + read.and_then(|mut target| { + let root = target.root.identity(); + target.receipts = receipts + .into_iter() + .filter(|receipt| { + receipt.evidence().root_identity == root && receipt.verify().is_ok() + }) + .collect(); + (!target.receipts.is_empty()).then_some(target) + }) + } Some(_) => None, }; ToolRun { @@ -2100,11 +2110,15 @@ impl Inner { scope: ReminderScope, ) -> Option { if def.scope != super::host::ToolScope::Builtin - || !matches!(def.spec.name.as_str(), "read" | "edit") + || !matches!(def.spec.name.as_str(), "read" | "edit" | "apply_patch") { return None; } - let path = invocation.input.get("path")?.as_str()?; + let path = invocation + .input + .get("path") + .and_then(Value::as_str) + .unwrap_or("."); let settings = self.instruction_settings(&invocation.directory).ok()?; let location = std::path::PathBuf::from(&invocation.directory); Some(FileTarget { @@ -2112,7 +2126,7 @@ impl Inner { location: location.clone(), scope, root: Arc::new(cyber_core::rules::RootBinding::capture(&location).ok()?), - receipt: None, + receipts: Vec::new(), operation_key: invocation.operation_key.clone(), attempt: invocation.attempt, }) @@ -2357,6 +2371,7 @@ fn settled( rule_reminders: Vec::new(), rule_read: None, rule_edit: None, + rule_patch: Vec::new(), structured_output: None, call_id: call_id.into(), status, @@ -2378,15 +2393,21 @@ pub(super) struct ToolRun { } impl ToolRun { - pub(super) fn operation_proof(&self) -> Option { - let target = self.read.as_ref()?; - Some(super::events::RuleOperation { - evidence: target.receipt.as_ref()?.evidence().clone(), - epoch: target.scope.epoch, - location: target.scope.directory.clone(), - operation_key: target.operation_key.clone(), - attempt: target.attempt, - }) + pub(super) fn operation_proof(&self) -> Vec { + let Some(target) = &self.read else { + return Vec::new(); + }; + target + .receipts + .iter() + .map(|receipt| super::events::RuleOperation { + evidence: receipt.evidence().clone(), + epoch: target.scope.epoch, + location: target.scope.directory.clone(), + operation_key: target.operation_key.clone(), + attempt: target.attempt, + }) + .collect() } /// Recheck under the settlement lock, after filesystem discovery. A Location /// switch or Epoch replacement during discovery must not consume new identities. @@ -2406,7 +2427,7 @@ impl ToolRun { } } - /// The target to load nested instructions for: only a read that succeeded. + /// Targets from a successful operation or the verified effects of a partial patch. pub(super) fn nested_target(&self) -> Option { let succeeded = matches!( self.outcome, @@ -2414,7 +2435,12 @@ impl ToolRun { | ToolOutcome::Structured { .. } | ToolOutcome::SkillSuggestions { failed: false, .. } ); - self.read.clone().filter(|_| succeeded) + self.read.clone().filter(|target| { + succeeded + || (self.def.spec.name == "apply_patch" + && !target.receipts.is_empty() + && patch_failure(&self.outcome)) + }) } } @@ -2454,17 +2480,13 @@ pub(super) struct FileTarget { pub file: std::path::PathBuf, pub scope: ReminderScope, pub root: Arc, - pub receipt: Option>, + pub receipts: Vec>, pub operation_key: String, pub attempt: u32, } impl FileTarget { fn valid_native(&self) -> bool { - self.root.verify().is_ok() - && self - .receipt - .as_ref() - .is_none_or(|receipt| receipt.verify().is_ok()) + self.root.verify().is_ok() && self.receipts.iter().all(|receipt| receipt.verify().is_ok()) } fn is_current(&self, state: &SessionState) -> bool { self.scope.is_current(state) && self.valid_native() @@ -2485,18 +2507,30 @@ impl FileTarget { if !self.valid_native() { return Default::default(); } - settings.scoped_rules = self.receipt.is_some(); + settings.scoped_rules = !self.receipts.is_empty(); settings.native_root = self - .receipt - .as_ref() + .receipts + .first() .map(|receipt| receipt.evidence().root_identity.clone()); - let loaded = super::instructions::load_nested( - global, - &self.location, - &self.file, - &settings, - attached, - ); + let loaded = if self.receipts.is_empty() { + super::instructions::load_nested( + global, + &self.location, + &self.file, + &settings, + attached, + ) + } else { + super::instructions::load_nested_targets( + global, + &self.location, + self.receipts + .iter() + .map(|receipt| receipt.evidence().path.as_path()), + &settings, + attached, + ) + }; if self.valid_native() { loaded } else { @@ -2508,25 +2542,41 @@ impl FileTarget { pub(super) fn with_operation_proof( mut settled: cyber_store::NewEvent, tool: &str, - proof: Option, + proof: Vec, ) -> cyber_store::NewEvent { - if settled - .data - .get("rule_reminders") - .and_then(serde_json::Value::as_array) - .is_some_and(|rules| !rules.is_empty()) - && let Some(proof) = proof + if proof.is_empty() + || !settled + .data + .get("rule_reminders") + .and_then(Value::as_array) + .is_some_and(|rules| !rules.is_empty()) { - let field = if tool == "edit" { - "rule_edit" - } else { - "rule_read" - }; - settled.data[field] = serde_json::to_value(proof).expect("rule read metadata serializes"); + return settled; + } + match tool { + "apply_patch" => { + settled.data["rule_patch"] = + serde_json::to_value(proof).expect("patch evidence serializes") + } + "edit" => { + settled.data["rule_edit"] = + serde_json::to_value(&proof[0]).expect("edit evidence serializes") + } + _ => { + settled.data["rule_read"] = + serde_json::to_value(&proof[0]).expect("read evidence serializes") + } } settled } +fn patch_failure(outcome: &ToolOutcome) -> bool { + matches!( + outcome, + ToolOutcome::Failed(_) | ToolOutcome::SkillSuggestions { failed: true, .. } + ) +} + /// `InstructionsLoaded` `load_reason` for files a read attached. pub(super) const NESTED_LOAD_REASON: &str = "nested_read"; @@ -2555,7 +2605,7 @@ impl Reminded { } /// Settle a call, appending reminders for matching skills and for `nested` instruction -/// files (attached only to a successful result). Also returns the instruction files that +/// files. A partial patch can attach instructions for its completed effects. Returns the files that /// entered the context, for `InstructionsLoaded`. The caller holds the Session lock and /// commits the resulting events as one transaction. pub(super) fn settlement_with_reminders( @@ -2570,6 +2620,7 @@ pub(super) fn settlement_with_reminders( Option, ) { let attach = !nested.is_empty(); + let partial_patch = attach && def.spec.name == "apply_patch" && patch_failure(&outcome); let (failed, output, value, skills) = match outcome { ToolOutcome::SkillSuggestions { failed, @@ -2577,6 +2628,7 @@ pub(super) fn settlement_with_reminders( value, skills, } => (failed, output, value, skills), + ToolOutcome::Failed(output) if partial_patch => (true, output, None, Vec::new()), ToolOutcome::Ok(output) if attach => (false, output, None, Vec::new()), ToolOutcome::Structured { output, value } if attach => { (false, output, Some(value), Vec::new()) @@ -2585,7 +2637,11 @@ pub(super) fn settlement_with_reminders( }; let (names, skill_reminder) = skill_reminder(skills, &seen.skills); let skill_reminder = skill_reminder.unwrap_or_default(); - let nested = if failed { None } else { Some(nested) }; + let nested = if failed && !partial_patch { + None + } else { + Some(nested) + }; let (output, instructions) = match with_reminders( output, nested, @@ -2636,6 +2692,7 @@ pub(super) fn settlement_with_reminders( rule_reminders: rule_bindings, rule_read: None, rule_edit: None, + rule_patch: Vec::new(), structured_output: value, call_id: call_id.into(), status: if failed { @@ -2737,6 +2794,7 @@ pub(super) fn settlement( rule_reminders: Vec::new(), rule_read: None, rule_edit: None, + rule_patch: Vec::new(), structured_output: None, call_id: call_id.into(), status: CallStatus::Ok, @@ -2772,6 +2830,7 @@ pub(super) fn settlement( rule_reminders: Vec::new(), rule_read: None, rule_edit: None, + rule_patch: Vec::new(), structured_output: Some(value), call_id: call_id.into(), status: CallStatus::Ok, @@ -2798,6 +2857,7 @@ pub(super) fn settlement( rule_reminders: Vec::new(), rule_read: None, rule_edit: None, + rule_patch: Vec::new(), structured_output: value, call_id: call_id.into(), status: if failed { @@ -2946,7 +3006,7 @@ mod skill_reminder_tests { file: "/repo/web/app.ts".into(), scope, root: root.clone(), - receipt: None, + receipts: Vec::new(), operation_key: String::new(), attempt: 1, }), @@ -2975,7 +3035,7 @@ mod skill_reminder_tests { file: "/repo/web/app.ts".into(), scope, root: root.clone(), - receipt: None, + receipts: Vec::new(), operation_key: String::new(), attempt: 1, }), @@ -3197,7 +3257,7 @@ mod skill_reminder_tests { epoch: 1, }, root: Arc::new(cyber_core::rules::RootBinding::capture(root_dir.path()).unwrap()), - receipt: None, + receipts: Vec::new(), operation_key: String::new(), attempt: 1, }; @@ -3278,7 +3338,7 @@ mod skill_reminder_tests { epoch: 1, }, root: Arc::new(cyber_core::rules::RootBinding::capture(root_dir.path()).unwrap()), - receipt: None, + receipts: Vec::new(), operation_key: String::new(), attempt: 1, }), diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 70cce9c77..55ca97a1e 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -394,6 +394,16 @@ impl RuleOperation { && self.evidence.root.to_str() == Some(binding.root.as_str()) && self.evidence.root.is_absolute() && self.evidence.path.starts_with(&self.evidence.root) + && (tool == "apply_patch" + || matches!( + self.evidence.target, + cyber_core::rules::OperationTarget::File { .. } + )) + && (tool == "apply_patch" + || binding + .operation_path + .as_ref() + .is_none_or(|path| path == &self.evidence.path)) } } @@ -413,6 +423,8 @@ pub struct ToolSettled { pub rule_read: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub rule_edit: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub rule_patch: Vec, #[serde( default, skip_serializing_if = "Option::is_none", diff --git a/crates/cyber-server/src/runtime/instructions.rs b/crates/cyber-server/src/runtime/instructions.rs index 2aec5613a..b603186b3 100644 --- a/crates/cyber-server/src/runtime/instructions.rs +++ b/crates/cyber-server/src/runtime/instructions.rs @@ -85,6 +85,9 @@ pub struct RuleBinding { pub generation: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub native_root: Option, + /// Physical operation target for this delivery. It is excluded from once-per-Epoch identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operation_path: Option, } impl RuleBinding { pub fn identity(&self) -> String { @@ -376,26 +379,45 @@ pub fn load_nested( settings: &InstructionSettings, attached: &BTreeSet, ) -> Instructions { - // Canonical on both sides, so a `..` or symlinked path cannot escape the project. - let (location, target) = (canonical(location), canonical(target)); - let dir = if target.is_dir() { - Some(target.as_path()) - } else { - target.parent() - }; - let dirs = dir - .map(|dir| project_dirs(&location, dir)) - .unwrap_or_default(); - if dirs.is_empty() { + load_nested_targets( + global_dir, + location, + std::iter::once(target), + settings, + attached, + ) +} + +pub(super) fn load_nested_targets<'a>( + global_dir: &Path, + location: &Path, + targets: impl IntoIterator, + settings: &InstructionSettings, + attached: &BTreeSet, +) -> Instructions { + let location = canonical(location); + let root = canonical(&cyber_core::config::project_root(&location)); + let targets: Vec<_> = targets + .into_iter() + .filter_map(|target| { + let target = canonical(target); + let directory = if target.is_dir() { + target.clone() + } else { + target.parent()?.to_path_buf() + }; + let dirs = project_dirs(&location, &directory); + (!dirs.is_empty()).then_some((target, directory, dirs)) + }) + .collect(); + if targets.is_empty() { return Instructions::default(); } let mut baseline = Loader::new(settings, settings.max_total_bytes); let _ = fill_baseline(&mut baseline, global_dir, &location); let budget = NESTED_MAX_TOTAL_BYTES.min(settings.max_total_bytes); let mut loader = Loader::new(settings, budget); - loader.rule_root = canonical(&cyber_core::config::project_root(&location)); - loader.rule_scope = dir.unwrap_or(location.as_path()).to_path_buf(); - loader.rule_target = Some(target.clone()); + loader.rule_root = root.clone(); loader.global_rules = canonical(global_dir).join("rules"); loader.delivered = baseline .out @@ -405,19 +427,25 @@ pub fn load_nested( .collect(); loader.delivered.extend(attached.iter().cloned()); loader.remaining = budget.saturating_sub(assembly::OPEN.len() + assembly::CLOSE.len()); - let root = canonical(&cyber_core::config::project_root(&location)); + let global_rules = loader.global_rules.clone(); - loader.rules(&global_rules, &root, Some(&target), false); - for dir in dirs { - if !settings.project { - break; - } - for file in directory_files(&dir, settings) { - if let Err(error) = loader.load(&file, Some(&root)) { - loader.out.warnings.push(error); + for (target, directory, dirs) in targets { + // Applicability depends on the target. Keep the byte, file and output + // budgets shared across all targets in the settlement. + loader.rule_seen.clear(); + loader.rule_scope = directory; + loader.rule_target = Some(target.clone()); + loader.rules(&global_rules, &root, Some(&target), false); + if settings.project { + for dir in dirs { + for file in directory_files(&dir, settings) { + if let Err(error) = loader.load(&file, Some(&root)) { + loader.out.warnings.push(error); + } + } + loader.project_rules(&dir, &root, Some(&target)); } } - loader.project_rules(&dir, &root, Some(&target)); } loader.out } @@ -618,6 +646,9 @@ impl<'a> Loader<'a> { digest: definition.digest.clone(), generation: generation.into(), native_root: self.settings.native_root.clone(), + operation_path: target + .filter(|_| self.settings.native_root.is_some()) + .map(Path::to_path_buf), }; let identity = binding.identity(); if !self.seen.insert(identity.clone()) { diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 7828e9330..c746c948e 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -2319,6 +2319,7 @@ impl Runtime { rule_reminders: Vec::new(), rule_read: None, rule_edit: None, + rule_patch: Vec::new(), structured_output: None, call_id: call_id.into(), status, diff --git a/crates/cyber-server/src/runtime/model.rs b/crates/cyber-server/src/runtime/model.rs index 05f2c6ac1..aae0f776e 100644 --- a/crates/cyber-server/src/runtime/model.rs +++ b/crates/cyber-server/src/runtime/model.rs @@ -603,6 +603,42 @@ pub(crate) const USER_SOURCES: [&str; 3] = ["user", super::MCP_PROMPT_SOURCE, "l /// Digest prefix of a Stop-hook continuation row; public digests are `sha256:`. pub(crate) const STOP_CONTINUATION_DIGEST: &str = "hook:Stop:"; +fn rule_operation_matches( + settled: &ToolSettled, + binding: &super::instructions::RuleBinding, + epoch: u32, + info: &SessionInfo, + call: &CallState, + stale: bool, +) -> bool { + if !settled.rule_patch.is_empty() { + return !stale + && binding.native_root.is_some() + && settled.rule_read.is_none() + && settled.rule_edit.is_none() + && binding.operation_path.as_ref().is_some_and(|target| { + settled + .rule_patch + .iter() + .any(|op| &op.evidence.path == target) + }) + && settled.rule_patch.len() <= cyber_core::rules::MAX_OPERATION_RECEIPTS + && settled + .rule_patch + .iter() + .all(|op| op.matches(binding, epoch, "apply_patch", info, call)); + } + if settled.status != CallStatus::Ok { + return false; + } + match (&binding.native_root, &settled.rule_read, &settled.rule_edit) { + (None, None, None) => true, + (Some(_), Some(read), None) => !stale && read.matches(binding, epoch, "read", info, call), + (Some(_), None, Some(edit)) => !stale && edit.matches(binding, epoch, "edit", info, call), + _ => false, + } +} + impl SessionState { pub(super) fn ensure_worktree_ready(&self) -> Result<(), super::RuntimeError> { if self.child_continuation_unknown { @@ -1506,41 +1542,23 @@ impl SessionState { if matches!(s.status, CallStatus::Ok | CallStatus::Error) && let Some(epoch) = &mut self.epoch { + epoch.reminded_rules.extend( + s.rule_reminders + .iter() + .filter(|binding| { + rule_operation_matches( + &s, + binding, + epoch.number, + &self.info, + call, + self.epoch_stale, + ) + }) + .map(super::instructions::RuleBinding::identity), + ); epoch.reminded_skills.extend(s.skill_reminders); epoch.reminded_instructions.extend(s.instruction_reminders); - if s.status == CallStatus::Ok { - epoch.reminded_rules.extend( - s.rule_reminders - .iter() - .filter(|binding| { - match (&binding.native_root, &s.rule_read, &s.rule_edit) { - (None, None, None) => true, - (Some(_), Some(read), None) => { - !self.epoch_stale - && read.matches( - binding, - epoch.number, - "read", - &self.info, - call, - ) - } - (Some(_), None, Some(edit)) => { - !self.epoch_stale - && edit.matches( - binding, - epoch.number, - "edit", - &self.info, - call, - ) - } - _ => false, - } - }) - .map(super::instructions::RuleBinding::identity), - ); - } } call.skill_activation = if s.status == CallStatus::Ok { s.skill_activation diff --git a/crates/cyber-tools/src/host.rs b/crates/cyber-tools/src/host.rs index f80d09016..9884cc9ef 100644 --- a/crates/cyber-tools/src/host.rs +++ b/crates/cyber-tools/src/host.rs @@ -1613,13 +1613,11 @@ impl ToolHost for BuiltinHost { Box::pin(async move { let read_receipt = FileReceiptSlot::default(); let outcome = self.execute_inner(inv, cancel, read_receipt.clone()).await; - let read = read_receipt - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(); + let files = + std::mem::take(&mut *read_receipt.lock().unwrap_or_else(PoisonError::into_inner)); cyber_server::runtime::ObservedExecution { outcome, - files: Some(read.into_iter().collect()), + files: Some(files), } }) } @@ -1967,7 +1965,10 @@ impl BuiltinHost { } // A hook retry owns a fresh attempt's observation; never carry a prior // read into a directory listing or an uninstrumented retried execution. - *read_receipt.lock().unwrap_or_else(PoisonError::into_inner) = None; + read_receipt + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clear(); Ok(Ctx { read_receipt, skill_paths: Default::default(), @@ -2056,7 +2057,7 @@ impl BuiltinHost { } /// Everything a tool needs for one call. -pub(crate) type FileReceiptSlot = Arc>>>; +pub(crate) type FileReceiptSlot = Arc>>>; pub(crate) struct Ctx<'a> { pub read_receipt: FileReceiptSlot, diff --git a/crates/cyber-tools/src/tools/fs.rs b/crates/cyber-tools/src/tools/fs.rs index 265b85985..6590ad83e 100644 --- a/crates/cyber-tools/src/tools/fs.rs +++ b/crates/cyber-tools/src/tools/fs.rs @@ -59,7 +59,7 @@ impl Tool for Read { .await .map_err(|_| failed("Observed file read failed"))? .map_err(|e| failed(format!("Could not read {}: {e}", path.display())))?; - (bytes, Some(std::sync::Arc::new(receipt))) + (bytes, Some(receipt)) } None => ( tokio::fs::read(&path) @@ -76,9 +76,9 @@ impl Tool for Read { if is_binary(&bytes) { return Err(failed("Binary file: application/octet-stream")); } - *ctx.read_receipt - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = receipt; + if let Some(receipt) = receipt { + record_file_receipt(ctx, receipt); + } ctx.host.mark_read(&ctx.inv.session_id, &path); ctx.note_skill_path(&path); if let Some(locations) = ctx.host.lsp.get() @@ -444,7 +444,7 @@ pub(crate) async fn guarded_write( .await .map_err(|e| failed(format!("Could not create {}: {e}", parent.display())))?; } - write_observed_edit(ctx, path, after, authorized.as_deref()) + write_observed(ctx, path, after, authorized.as_deref(), false) .await .map_err(|e| failed(format!("Could not write {}: {e}", path.display())))?; ctx.host.mark_read(&ctx.inv.session_id, path); @@ -454,47 +454,109 @@ pub(crate) async fn guarded_write( Ok(super::intelligence::feedback(ctx, path, after, origin).await) } -async fn write_observed_edit( +enum WriteObservation { + Update(cyber_core::rules::ObservedFile), + Create(cyber_core::rules::PendingCreation), +} +impl WriteObservation { + fn after_write(self, file: &std::fs::File) -> Result { + match self { + Self::Update(observation) => observation.after_write(file), + Self::Create(observation) => observation.after_write(file), + } + } +} + +pub(super) async fn write_observed( ctx: &Ctx<'_>, path: &Path, bytes: &[u8], authorized: Option<&Path>, + creating: bool, ) -> std::io::Result<()> { // An edit approval for an alias does not attest a separately named resource. let observation = authorized - .filter(|_| ctx.inv.name == "edit") - .filter(|target| { - ctx.location - .canonicalize() - .ok() - .and_then(|location| { - target - .strip_prefix(location) - .ok() - .map(|relative| ctx.location.join(relative)) - }) - .as_deref() - == Some(path) - }) - .and_then(|target| cyber_core::rules::ObservedFile::open(&ctx.location, path, target).ok()); + .filter(|_| matches!(ctx.inv.name.as_str(), "edit" | "apply_patch")) + .filter(|target| workspace_spelling_matches(ctx, path, target)) + .and_then(|target| { + if creating { + cyber_core::rules::PendingCreation::capture(&ctx.location, path, target) + .ok() + .map(WriteObservation::Create) + } else { + cyber_core::rules::ObservedFile::open(&ctx.location, path, target) + .ok() + .map(WriteObservation::Update) + } + }); let Some(observation) = observation else { return tokio::fs::write(path, bytes).await; }; use tokio::io::AsyncWriteExt; - let mut file = tokio::fs::File::create(path).await?; + let mut file = if creating { + tokio::fs::File::options() + .write(true) + .create_new(true) + .open(path) + .await? + } else { + tokio::fs::File::create(path).await? + }; file.write_all(bytes).await?; file.flush().await?; let file = file.into_std().await; let receipt = tokio::task::spawn_blocking(move || observation.after_write(&file)).await; if let Ok(Ok(receipt)) = receipt { - *ctx.read_receipt - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(std::sync::Arc::new(receipt)); + record_file_receipt(ctx, receipt); + } + Ok(()) +} + +pub(super) async fn remove_observed( + ctx: &Ctx<'_>, + path: &Path, + authorized: &Path, +) -> std::io::Result<()> { + let observed = (ctx.inv.name == "apply_patch" + && workspace_spelling_matches(ctx, path, authorized)) + .then(|| cyber_core::rules::PendingRemoval::capture(&ctx.location, path, authorized).ok()) + .flatten(); + let Some(observed) = observed else { + return tokio::fs::remove_file(path).await; + }; + let receipt = tokio::task::spawn_blocking(move || observed.remove()) + .await + .map_err(std::io::Error::other)??; + if let Some(receipt) = receipt { + record_file_receipt(ctx, receipt); } Ok(()) } +fn workspace_spelling_matches(ctx: &Ctx<'_>, path: &Path, target: &Path) -> bool { + ctx.location + .canonicalize() + .ok() + .and_then(|location| { + target + .strip_prefix(location) + .ok() + .map(|relative| ctx.location.join(relative)) + }) + .as_deref() + == Some(path) +} + +fn record_file_receipt(ctx: &Ctx<'_>, receipt: cyber_core::rules::FileReceipt) { + let mut files = ctx + .read_receipt + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if files.len() < cyber_core::rules::MAX_OPERATION_RECEIPTS { + files.push(std::sync::Arc::new(receipt)); + } +} + pub(crate) async fn authorize_edit( ctx: &Ctx<'_>, path: &Path, diff --git a/crates/cyber-tools/src/tools/patch.rs b/crates/cyber-tools/src/tools/patch.rs index 585de7798..c868a7e84 100644 --- a/crates/cyber-tools/src/tools/patch.rs +++ b/crates/cyber-tools/src/tools/patch.rs @@ -408,9 +408,13 @@ pub(crate) async fn apply( file_edit: true, ..Request::default() }; + let authorized: Vec<_> = changes + .iter() + .map(|change| crate::host::canonical(&change.path)) + .collect(); ctx.authorize_as("apply_patch", req, resources, json!({ "diff": diff })) .await?; - write_all(ctx, &changes).await + write_all(ctx, &changes, &authorized).await } /// A patch may not add a redaction marker; one moved or kept with its file is no addition. @@ -435,13 +439,17 @@ fn refuse_markers(changes: &[Change]) -> Result<(), ToolError> { crate::host::refuse_marker(&before, &after) } -async fn write_all(ctx: &Ctx<'_>, changes: &[Change]) -> Result { +async fn write_all( + ctx: &Ctx<'_>, + changes: &[Change], + authorized: &[PathBuf], +) -> Result { let mut applied: Vec = Vec::new(); let mut pending = Vec::new(); - for change in changes { + for (change, target) in changes.iter().zip(authorized) { let label = format!("{} {}", change.kind, ctx.resource(&change.path)); let origin = super::intelligence::capture(ctx, &change.path); - if let Err(e) = write_one(ctx, change).await { + if let Err(e) = write_one(ctx, change, target).await { let message = format!( "Patch partially applied before failing at {}: {e}. Applied: {}", ctx.resource(&change.path), @@ -482,7 +490,7 @@ async fn patch_feedback( results.concat() } -async fn write_one(ctx: &Ctx<'_>, change: &Change) -> Result<(), String> { +async fn write_one(ctx: &Ctx<'_>, change: &Change, authorized: &Path) -> Result<(), String> { let workspace = cyber_server::workspace::mutation_lock(&ctx.location).map_err(|error| error.to_string())?; let _workspace = workspace.lock_owned().await; @@ -498,12 +506,18 @@ async fn write_one(ctx: &Ctx<'_>, change: &Change) -> Result<(), String> { .await .map_err(|e| e.to_string())?; } - tokio::fs::write(&change.path, bytes) - .await - .map_err(|e| e.to_string())?; + super::fs::write_observed( + ctx, + &change.path, + bytes, + Some(authorized), + change.before.is_none(), + ) + .await + .map_err(|e| e.to_string())?; ctx.host.mark_read(&ctx.inv.session_id, &change.path); } - None => tokio::fs::remove_file(&change.path) + None => super::fs::remove_observed(ctx, &change.path, authorized) .await .map_err(|e| e.to_string())?, } diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 71dd0d3e8..3eb838237 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads and edits use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native read or edit and dispatched attempt. Finish patch and partial-effect delivery, exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads, edits and direct patches use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native operation and dispatched attempt. A partial patch can deliver rules for completed files. Finish shell-routed patches, exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 51ad0a25e..ea0a4cd75 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1221,9 +1221,9 @@ Project rule bytes are included in the existing checkout trust digest. Review wi Definition reads use the fenced native reader: linked files/directories, hard links and special files are refused. The reader caps a definition at 1 MiB; frontmatter is limited to 16 KiB, with up to 64 relative patterns of 256 bytes each. A project inventory covers up to 128 definitions and 4 MiB, with bounded directory enumeration. These review/discovery bounds are separate from the smaller rendered instruction budgets. Unknown inventories grant no rule authority. `cyber doctor` reports held, refused and truncated baseline rules. -Scoped rules activate after a successful built-in file read or edit with a native receipt. Rule matching uses the physical file from the completed operation after authorization and any hook rewrite. Edit receipts require the handle that completed the write to match the observed file. A directory listing, unsupported observation, or changed root/target binding does not consume delivery. The tool's own result is preserved. Ordinary instruction behavior remains available to legacy hosts without native receipts; those hosts cannot activate path-scoped rules. +Scoped rules activate after a successful built-in read, edit, or file change from `apply_patch` with a native receipt. Rule matching uses the physical file from the completed operation after authorization and any hook rewrite. Writes are bound to the handle that completed the effect. Patch deletions bind the physical parent directory and removed name. A partially applied patch keeps its error status and can deliver rules for its verified completed files. A directory listing, unsupported observation, or changed root/target binding does not consume delivery. The tool's own result is preserved. Ordinary instruction behavior remains available to legacy hosts without native receipts; those hosts cannot activate path-scoped rules. -The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Patch and partial-settlement delivery, exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. +The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Patch rule discovery shares the reminder budget across up to 32 verified file effects. Shell-routed patches, exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index 09e65b567..bbc8b2859 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -28,7 +28,7 @@ Definition discovery bounds are 128 project files, 4096 examined entries per rul Nested read discovery resolves instruction settings after a successful effect. Its captured Session Location and Context Epoch must still be active before discovery and again under the settlement lock. An absent/stale/replaced Epoch or changed Location suppresses the reminder without recording delivery; normal group settlement and exact `/approve` replay share this check. The captured read also retains the native directory identities from the volume root through the physical Location, verifies the project-root boundary, and rechecks those bindings before/after discovery and under settlement. Unix retains directory handles to prevent inode reuse; Windows releases handles after capture and verifies full native IDs, so the observation does not freeze editor/worktree renames. Ordinary file content edits do not invalidate the root. Replacement/removal, changed ancestors, linked substitution, a retargeted declared Location alias or a changed project boundary withhold the reminder. These point-in-time context/root checks do not prove which file handle performed the effect: actual per-file operation receipts (including partial writes), exact configuration-generation fencing through commit and revocation withdrawal remain required. -Built-in reads now read through an observed file handle after authorization and any input rewrite, and hand the resulting receipt to the runtime through the host execution boundary. Workspace read targets preserve the normal size and hard-link behavior; rule-definition and memory-storage source readers keep their stricter checks. The receipt retains actual native file/ancestor identities, a physical path and the captured project root. Unsupported observations keep the read result but grant no scoped-rule delivery. Rechecking a receipt reads metadata, not file contents. The ordinary legacy host path cannot activate scoped definitions. Successful labelled delivery stores `rule_read` atomically with `rule_reminders`: actual target/root IDs, captured Location/Epoch, operation key and dispatched attempt. Version-2 rule IDs incorporate the native root; old IDs and legacy records replay unchanged. The recorded proof describes an earlier operation and never grants new filesystem authority. Replay checks the dispatch and context before folding new identities. An existing-file edit compares the completed writer handle with the observed native file and records `rule_edit` under the same dispatch and Context Epoch checks. A failed read leaves its scoped rule eligible for that edit. Patch receipts, positive partial effects, exact configuration generation and revocation/restart acceptance remain open. +Built-in reads now read through an observed file handle after authorization and any input rewrite, and hand the resulting receipt to the runtime through the host execution boundary. Workspace read targets preserve the normal size and hard-link behavior; rule-definition and memory-storage source readers keep their stricter checks. The receipt retains actual native file/ancestor identities, a physical path and the captured project root. Unsupported observations keep the read result but grant no scoped-rule delivery. Rechecking a receipt reads metadata, not file contents. The ordinary legacy host path cannot activate scoped definitions. Successful labelled delivery stores `rule_read` atomically with `rule_reminders`: actual target/root IDs, captured Location/Epoch, operation key and dispatched attempt. Version-2 rule IDs incorporate the native root; old IDs and legacy records replay unchanged. The recorded proof describes an earlier operation and never grants new filesystem authority. Replay checks the dispatch and context before folding new identities. An existing-file edit compares the completed writer handle with the observed native file and records `rule_edit` under the same dispatch and Context Epoch checks. A failed read leaves its scoped rule eligible for that edit. Direct patches collect up to 32 completed native effects in execution order. Each delivered rule records its matching physical target and the settlement stores the effect receipts under `rule_patch`. The shared discovery and rendering budgets apply to the whole patch. Creation uses the completed exclusive writer and bound parent. Removal executes relative to the observed parent directory and attests the successful removal of its name and the resulting absence. A later patch failure preserves the error status and retains the completed effects. Shell-routed patches, exact configuration generation and revocation/restart acceptance remain open. Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index d98871259..fd9745542 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -76,3 +76,12 @@ - **AND** the settlement SHALL bind any delivered rule to that read's native root/file identities, Location, Context Epoch, operation key and dispatched attempt - **AND** a listing, denied operation or successful result without eligible native evidence SHALL NOT consume scoped rule delivery - **AND** replay SHALL NOT consume a new rule identity from a receipt mismatched to the dispatch, root, Location or Epoch + + +#### Scenario: A patch fails after a completed file change +- **GIVEN** a direct patch has several authorized file changes +- **WHEN** the first change completes and a later precondition fails +- **THEN** the patch SHALL retain its error status +- **AND** rule discovery SHALL consider the verified completed effects under the shared reminder budget +- **AND** files whose effects failed or never ran SHALL NOT supply or consume rule reminders +- **AND** replay SHALL associate each delivered rule with a receipt for its actual operation target, Location, dispatched attempt and Context Epoch diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 9ed4b2bd7..ff65b5f75 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1812,3 +1812,26 @@ configuration generation and revocation/restart acceptance remain open. Local edit verification passed 12 filtered Core tests, 17 App instruction tests, and all-target Clippy for Core, Tools, Server and App. Formatting, SDK consistency and strict OpenSpec checks passed. Windows and platform acceptance remain pending. + +Direct patch continuation. A patch now retains up to 32 verified completed file +effects and loads their rules under one discovery and reminder budget. Updates +and exclusive creations bind the completed writer handle. A deletion records +the pinned native parent and removed name after a successful relative unlink and +verified absence. This evidence describes the namespace operation. The patch +keeps its error status after a later failure. Only completed targets can deliver +rules and each labelled binding retains its matching path. Historical read and +edit settlement fields still decode. Shell-routed patches, exact configuration +generation, revocation and restart acceptance remain open. + +The App regression covers updates, creation under a new parent and deletion, +plus a later precondition failure that prevents the remaining effects. It checks +the actual tool output, recorded paths, error status and replay identity. Replay +does not mark a rule delivered when its matching receipt names another target. +The Core test confirms a replaced parent prevents the observed removal. + +Local patch verification passed 13 filtered Core tests, 18 App instruction tests, +455 Server library tests (one ignored subprocess probe), and all-target Clippy +for Core, Tools, Server and App. Strict OpenSpec validated all 100 items. Spec +lint reported zero errors and 25 warnings. Formatting and generated SDK +consistency passed. Windows, full workspace and SDK runtime suites were not +rerun locally. Current-source native acceptance remains pending. From a4d28df4d6bc9ddd79330f560ecd81b835fb2d37 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 19:00:15 -0400 Subject: [PATCH 82/87] Carry scoped rule receipts through intercepted shell patches --- .github/workflows/non-git-snapshot-proof.yml | 2 + crates/cyber-app/tests/instructions.rs | 45 +++++++++++++--- .../cyber-server/src/runtime/auto_override.rs | 2 +- crates/cyber-server/src/runtime/drain.rs | 29 +++++----- crates/cyber-server/src/runtime/events.rs | 11 +++- crates/cyber-server/src/runtime/model.rs | 10 +++- crates/cyber-tools/src/tools/fs.rs | 12 ++--- crates/cyber-tools/tests/shell_patch.rs | 53 ++++++++++++++----- docs/goal-delivery.md | 2 +- docs/local-core-guide.md | 2 +- .../complete-daily-coding-surfaces/design.md | 2 +- .../specs/system-context/spec.md | 2 +- .../complete-daily-coding-surfaces/tasks.md | 18 +++++++ 13 files changed, 143 insertions(+), 47 deletions(-) diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 665783132..7f4d87840 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -40,6 +40,7 @@ on: - crates/cyber-server/src/runtime/auto_override.rs - crates/cyber-tools/tests/instruction_config.rs - crates/cyber-tools/tests/path_skills.rs + - crates/cyber-tools/tests/shell_patch.rs - crates/cyber-server/src/runtime/context.rs - crates/cyber-server/src/runtime/instructions.rs - crates/cyber-server/src/runtime/instructions/** @@ -214,6 +215,7 @@ jobs: cargo test --locked -p cyber-app --features native-git-acceptance --test instructions -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-tools --test instruction_config -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-tools --test path_skills -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-tools --test shell_patch -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-cli --features native-git-acceptance --bin cyber commands::doctor::instruction_tests:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log - name: Retain native evidence if: always() diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index 702f46f9e..6f2e04f49 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -38,9 +38,17 @@ fn reply(body: &Value) -> Value { patch.push_str( "*** Add File: assets/new/c.txt\n+created-c\n*** Delete File: old/d.txt\n*** End Patch", ); - let arguments = json!({"patch":patch}).to_string(); + let (name, input) = if text.contains("via shell") { + ( + "bash", + json!({"command":format!("apply_patch <<'PATCH'\n{patch}\nPATCH\n")}), + ) + } else { + ("apply_patch", json!({"patch":patch})) + }; + let arguments = input.to_string(); return json!({"tool_calls":[{"index":0,"id":"call_patch","type":"function", - "function":{"name":"apply_patch","arguments":arguments}}]}); + "function":{"name":name,"arguments":arguments}}]}); } if text.contains("edit the app") { let arguments = json!({"path":"web/src/app.ts","old_string":"const version = 1","new_string":"const version = 2"}).to_string(); @@ -974,8 +982,16 @@ async fn denied_read_leaves_rule_available_for_a_successful_edit() { #[tokio::test] async fn patch_rules_follow_completed_files_even_after_a_later_failure() { use cyber_server::runtime::CallStatus; - for partial in [true, false] { - let f = Fixture::start_with_model(json!({}), true).await; + for (shell, partial) in [(false, true), (false, false), (true, true), (true, false)] { + if shell && !cfg!(unix) { + continue; // PowerShell interception has its own host test. + } + let config = if shell { + json!({"sandbox":{"policy":"workspace-write","env":{"set":{"PATH":"/usr/bin:/bin"}}}}) + } else { + json!({}) + }; + let f = Fixture::start_with_model(config, !shell).await; f.write("web/a.txt", "before-a\n"); f.write("db/b.txt", "before-b\n"); f.write( @@ -997,12 +1013,16 @@ async fn patch_rules_follow_completed_files_even_after_a_later_failure() { ); f.approve(&f.repo); let id = f.session(&f.repo).await; - let prompt = if partial { + let mut prompt = if partial { "patch the app partially" } else { "patch the app" - }; - f.prompt(&id, &f.repo, prompt, 2).await; + } + .to_owned(); + if shell { + prompt.push_str(" via shell"); + } + f.prompt(&id, &f.repo, &prompt, 2).await; let result = last_tool_result(&f.last_request()); assert!(result.contains("PATCH_WEB_RULE"), "{result}"); @@ -1040,6 +1060,17 @@ async fn patch_rules_follow_completed_files_even_after_a_later_failure() { .iter() .find(|event| event.data["rule_patch"].is_array()) .unwrap(); + if shell { + assert_eq!(settled.data["metadata"]["intercepted"], "apply_patch"); + let mut unmarked = events.clone(); + let event = unmarked + .iter_mut() + .find(|event| event.id == settled.id) + .unwrap(); + event.data.as_object_mut().unwrap().remove("metadata"); + let replay = cyber_server::runtime::SessionState::replay(&unmarked).unwrap(); + assert!(replay.epoch.unwrap().reminded_rules.is_empty()); + } let proof = settled.data["rule_patch"].as_array().unwrap(); assert_eq!(proof.len(), ids.len()); if !partial { diff --git a/crates/cyber-server/src/runtime/auto_override.rs b/crates/cyber-server/src/runtime/auto_override.rs index 0e184f58f..87d230320 100644 --- a/crates/cyber-server/src/runtime/auto_override.rs +++ b/crates/cyber-server/src/runtime/auto_override.rs @@ -506,7 +506,7 @@ impl Inner { loaded.into_iter().collect(), match run.def.spec.name.as_str() { "edit" => "nested_edit", - "apply_patch" => "nested_patch", + tool if super::events::is_patch_carrier(tool) => "nested_patch", _ => super::drain::NESTED_LOAD_REASON, }, ) diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index ab71166e5..f9e695eec 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -1838,7 +1838,7 @@ impl Inner { events.push(self.with_call_metadata(&session_id, &run.call_id, &directory, settled)); match run.def.spec.name.as_str() { "edit" => edited.extend(instructions), - "apply_patch" => patched.extend(instructions), + tool if super::events::is_patch_carrier(tool) => patched.extend(instructions), _ => loaded.extend(instructions), } stops.extend(self.take_call_hook_effects(&session_id, &run.call_id, &mut events)); @@ -2079,7 +2079,7 @@ impl Inner { Some(receipts) if !receipts.is_empty() && receipts.len() <= cyber_core::rules::MAX_OPERATION_RECEIPTS - && (def.spec.name == "apply_patch" || receipts.len() == 1) => + && (super::events::is_patch_carrier(&def.spec.name) || receipts.len() == 1) => { read.and_then(|mut target| { let root = target.root.identity(); @@ -2110,19 +2110,21 @@ impl Inner { scope: ReminderScope, ) -> Option { if def.scope != super::host::ToolScope::Builtin - || !matches!(def.spec.name.as_str(), "read" | "edit" | "apply_patch") + || !(matches!(def.spec.name.as_str(), "read" | "edit") + || super::events::is_patch_carrier(&def.spec.name)) { return None; } - let path = invocation - .input - .get("path") - .and_then(Value::as_str) - .unwrap_or("."); - let settings = self.instruction_settings(&invocation.directory).ok()?; let location = std::path::PathBuf::from(&invocation.directory); + let file = if def.spec.name == "read" { + let path = invocation.input.get("path")?.as_str()?; + let settings = self.instruction_settings(&invocation.directory).ok()?; + super::instructions::read_target(&location, settings.home.as_deref(), path) + } else { + location.clone() + }; Some(FileTarget { - file: super::instructions::read_target(&location, settings.home.as_deref(), path), + file, location: location.clone(), scope, root: Arc::new(cyber_core::rules::RootBinding::capture(&location).ok()?), @@ -2437,7 +2439,7 @@ impl ToolRun { ); self.read.clone().filter(|target| { succeeded - || (self.def.spec.name == "apply_patch" + || (super::events::is_patch_carrier(&self.def.spec.name) && !target.receipts.is_empty() && patch_failure(&self.outcome)) }) @@ -2554,7 +2556,7 @@ pub(super) fn with_operation_proof( return settled; } match tool { - "apply_patch" => { + tool if super::events::is_patch_carrier(tool) => { settled.data["rule_patch"] = serde_json::to_value(proof).expect("patch evidence serializes") } @@ -2620,7 +2622,8 @@ pub(super) fn settlement_with_reminders( Option, ) { let attach = !nested.is_empty(); - let partial_patch = attach && def.spec.name == "apply_patch" && patch_failure(&outcome); + let partial_patch = + attach && super::events::is_patch_carrier(&def.spec.name) && patch_failure(&outcome); let (failed, output, value, skills) = match outcome { ToolOutcome::SkillSuggestions { failed, diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 55ca97a1e..2e06e49cd 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -374,6 +374,13 @@ pub struct RuleOperation { pub operation_key: String, pub attempt: u32, } +/// Shell tools can carry patch receipts only when the host intercepts the call. +pub(super) fn is_patch_carrier(tool: &str) -> bool { + ["apply_patch", "bash", "powershell"] + .iter() + .any(|name| tool.eq_ignore_ascii_case(name)) +} + impl RuleOperation { pub fn matches( &self, @@ -394,12 +401,12 @@ impl RuleOperation { && self.evidence.root.to_str() == Some(binding.root.as_str()) && self.evidence.root.is_absolute() && self.evidence.path.starts_with(&self.evidence.root) - && (tool == "apply_patch" + && (is_patch_carrier(tool) || matches!( self.evidence.target, cyber_core::rules::OperationTarget::File { .. } )) - && (tool == "apply_patch" + && (is_patch_carrier(tool) || binding .operation_path .as_ref() diff --git a/crates/cyber-server/src/runtime/model.rs b/crates/cyber-server/src/runtime/model.rs index aae0f776e..2e866c5d1 100644 --- a/crates/cyber-server/src/runtime/model.rs +++ b/crates/cyber-server/src/runtime/model.rs @@ -612,7 +612,15 @@ fn rule_operation_matches( stale: bool, ) -> bool { if !settled.rule_patch.is_empty() { + let intercepted = settled + .metadata + .as_ref() + .and_then(|metadata| metadata.get("intercepted")) + .and_then(Value::as_str) + == Some("apply_patch"); return !stale + && (call.name.eq_ignore_ascii_case("apply_patch") + || (is_patch_carrier(&call.name) && intercepted)) && binding.native_root.is_some() && settled.rule_read.is_none() && settled.rule_edit.is_none() @@ -626,7 +634,7 @@ fn rule_operation_matches( && settled .rule_patch .iter() - .all(|op| op.matches(binding, epoch, "apply_patch", info, call)); + .all(|op| op.matches(binding, epoch, &call.name, info, call)); } if settled.status != CallStatus::Ok { return false; diff --git a/crates/cyber-tools/src/tools/fs.rs b/crates/cyber-tools/src/tools/fs.rs index 6590ad83e..87f7ce003 100644 --- a/crates/cyber-tools/src/tools/fs.rs +++ b/crates/cyber-tools/src/tools/fs.rs @@ -425,7 +425,9 @@ pub(crate) async fn guarded_write( )?; ctx.check_external_writes(&[path.to_path_buf()]).await?; // Capture the physical spelling before permission can suspend the call. - let authorized = path.canonicalize().ok(); + let authorized = (ctx.inv.name == "edit") + .then(|| path.canonicalize().ok()) + .flatten(); authorize_edit(ctx, path, diff).await?; let origin = super::intelligence::capture(ctx, path); let workspace = cyber_server::workspace::mutation_lock(&ctx.location) @@ -476,7 +478,6 @@ pub(super) async fn write_observed( ) -> std::io::Result<()> { // An edit approval for an alias does not attest a separately named resource. let observation = authorized - .filter(|_| matches!(ctx.inv.name.as_str(), "edit" | "apply_patch")) .filter(|target| workspace_spelling_matches(ctx, path, target)) .and_then(|target| { if creating { @@ -517,10 +518,9 @@ pub(super) async fn remove_observed( path: &Path, authorized: &Path, ) -> std::io::Result<()> { - let observed = (ctx.inv.name == "apply_patch" - && workspace_spelling_matches(ctx, path, authorized)) - .then(|| cyber_core::rules::PendingRemoval::capture(&ctx.location, path, authorized).ok()) - .flatten(); + let observed = workspace_spelling_matches(ctx, path, authorized) + .then(|| cyber_core::rules::PendingRemoval::capture(&ctx.location, path, authorized).ok()) + .flatten(); let Some(observed) = observed else { return tokio::fs::remove_file(path).await; }; diff --git a/crates/cyber-tools/tests/shell_patch.rs b/crates/cyber-tools/tests/shell_patch.rs index 5c98e8e07..fb75881b8 100644 --- a/crates/cyber-tools/tests/shell_patch.rs +++ b/crates/cyber-tools/tests/shell_patch.rs @@ -79,10 +79,15 @@ async fn a_tool_scoped_apply_patch_deny_blocks_the_shell_patch() { "rules": [{"action": "edit", "resource": "*", "effect": "deny", "tool": "apply_patch"}] }})); f.write("a.txt", "old\n"); - let out = failed( - f.call("default", "bash", json!({"command": heredoc("")})) - .await, - ); + let execution = f + .host + .execute_observed( + f.invocation("default", "bash", json!({"command": heredoc("")})), + tokio_util::sync::CancellationToken::new(), + ) + .await; + assert!(execution.files.unwrap().is_empty()); + let out = failed(execution.outcome); assert!(!out.is_empty()); assert_eq!(f.read("a.txt"), "old\n"); } @@ -95,9 +100,21 @@ async fn a_powershell_here_string_runs_as_apply_patch() { let f = Fixture::new(); f.write("a.txt", "old\n"); let script = format!("cd . && apply_patch @'\n{PATCH}\n'@"); - let out = ok(f - .call("accept-edits", "powershell", json!({"command": script})) - .await); + let execution = f + .host + .execute_observed( + f.invocation("accept-edits", "powershell", json!({"command": script})), + tokio_util::sync::CancellationToken::new(), + ) + .await; + let files = execution.files.unwrap(); + assert_eq!(files.len(), 1); + assert_eq!( + files[0].evidence().path, + f.repo.join("a.txt").canonicalize().unwrap() + ); + files[0].verify().unwrap(); + let out = ok(execution.outcome); assert_eq!(out, "M a.txt"); assert_eq!(f.read("a.txt"), "new\n"); assert_eq!( @@ -124,11 +141,16 @@ async fn an_installed_apply_patch_program_is_left_alone() { ))) .unwrap(); f.env.set("PATH", path.to_str().unwrap()); - let _ = f - .call("default", "bash", json!({"command": heredoc("")})) + let execution = f + .host + .execute_observed( + f.invocation("default", "bash", json!({"command": heredoc("")})), + tokio_util::sync::CancellationToken::new(), + ) .await; + assert!(execution.files.unwrap().is_empty()); assert_eq!(intercepted(&f, "call_bash"), None); - assert_eq!(f.read("a.txt"), "old\n", "our patch tool did not run"); + // The external program may change the file. Only built-in routing can mint receipts. } /// Regression: PATH comes from the shell's environment, so an `apply_patch` program @@ -170,11 +192,16 @@ async fn a_login_shell_runs_the_script_as_written() { "sandbox": {"env": {"login_shell": true}} })); f.write("a.txt", "old\n"); - let _ = f - .call("default", "bash", json!({"command": heredoc("")})) + let execution = f + .host + .execute_observed( + f.invocation("default", "bash", json!({"command": heredoc("")})), + tokio_util::sync::CancellationToken::new(), + ) .await; + assert!(execution.files.unwrap().is_empty()); assert_eq!(intercepted(&f, "call_bash"), None); - assert_eq!(f.read("a.txt"), "old\n"); + // Login profiles can make an external apply_patch program available. } /// "Long build log for an OpenAI model": a middle profile keeps about 20 KB of each diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 3eb838237..f66128dc3 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,7 +21,7 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads, edits and direct patches use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native operation and dispatched attempt. A partial patch can deliver rules for completed files. Finish shell-routed patches, exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads, edits and patches, including intercepted shell patches, use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native operation and dispatched attempt. A partial patch can deliver rules for completed files. Finish exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index ea0a4cd75..444489c76 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1223,7 +1223,7 @@ Definition reads use the fenced native reader: linked files/directories, hard li Scoped rules activate after a successful built-in read, edit, or file change from `apply_patch` with a native receipt. Rule matching uses the physical file from the completed operation after authorization and any hook rewrite. Writes are bound to the handle that completed the effect. Patch deletions bind the physical parent directory and removed name. A partially applied patch keeps its error status and can deliver rules for its verified completed files. A directory listing, unsupported observation, or changed root/target binding does not consume delivery. The tool's own result is preserved. Ordinary instruction behavior remains available to legacy hosts without native receipts; those hosts cannot activate path-scoped rules. -The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Patch rule discovery shares the reminder budget across up to 32 verified file effects. Shell-routed patches, exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. +The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Patch rule discovery shares the reminder budget across up to 32 verified file effects. Patches intercepted by the built-in Bash or PowerShell path use the same receipts. Shell settlements record the interception and replay requires it. Exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/design.md b/openspec/changes/complete-daily-coding-surfaces/design.md index bbc8b2859..791e15f5f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/design.md +++ b/openspec/changes/complete-daily-coding-surfaces/design.md @@ -28,7 +28,7 @@ Definition discovery bounds are 128 project files, 4096 examined entries per rul Nested read discovery resolves instruction settings after a successful effect. Its captured Session Location and Context Epoch must still be active before discovery and again under the settlement lock. An absent/stale/replaced Epoch or changed Location suppresses the reminder without recording delivery; normal group settlement and exact `/approve` replay share this check. The captured read also retains the native directory identities from the volume root through the physical Location, verifies the project-root boundary, and rechecks those bindings before/after discovery and under settlement. Unix retains directory handles to prevent inode reuse; Windows releases handles after capture and verifies full native IDs, so the observation does not freeze editor/worktree renames. Ordinary file content edits do not invalidate the root. Replacement/removal, changed ancestors, linked substitution, a retargeted declared Location alias or a changed project boundary withhold the reminder. These point-in-time context/root checks do not prove which file handle performed the effect: actual per-file operation receipts (including partial writes), exact configuration-generation fencing through commit and revocation withdrawal remain required. -Built-in reads now read through an observed file handle after authorization and any input rewrite, and hand the resulting receipt to the runtime through the host execution boundary. Workspace read targets preserve the normal size and hard-link behavior; rule-definition and memory-storage source readers keep their stricter checks. The receipt retains actual native file/ancestor identities, a physical path and the captured project root. Unsupported observations keep the read result but grant no scoped-rule delivery. Rechecking a receipt reads metadata, not file contents. The ordinary legacy host path cannot activate scoped definitions. Successful labelled delivery stores `rule_read` atomically with `rule_reminders`: actual target/root IDs, captured Location/Epoch, operation key and dispatched attempt. Version-2 rule IDs incorporate the native root; old IDs and legacy records replay unchanged. The recorded proof describes an earlier operation and never grants new filesystem authority. Replay checks the dispatch and context before folding new identities. An existing-file edit compares the completed writer handle with the observed native file and records `rule_edit` under the same dispatch and Context Epoch checks. A failed read leaves its scoped rule eligible for that edit. Direct patches collect up to 32 completed native effects in execution order. Each delivered rule records its matching physical target and the settlement stores the effect receipts under `rule_patch`. The shared discovery and rendering budgets apply to the whole patch. Creation uses the completed exclusive writer and bound parent. Removal executes relative to the observed parent directory and attests the successful removal of its name and the resulting absence. A later patch failure preserves the error status and retains the completed effects. Shell-routed patches, exact configuration generation and revocation/restart acceptance remain open. +Built-in reads now read through an observed file handle after authorization and any input rewrite, and hand the resulting receipt to the runtime through the host execution boundary. Workspace read targets preserve the normal size and hard-link behavior; rule-definition and memory-storage source readers keep their stricter checks. The receipt retains actual native file/ancestor identities, a physical path and the captured project root. Unsupported observations keep the read result but grant no scoped-rule delivery. Rechecking a receipt reads metadata, not file contents. The ordinary legacy host path cannot activate scoped definitions. Successful labelled delivery stores `rule_read` atomically with `rule_reminders`: actual target/root IDs, captured Location/Epoch, operation key and dispatched attempt. Version-2 rule IDs incorporate the native root; old IDs and legacy records replay unchanged. The recorded proof describes an earlier operation and never grants new filesystem authority. Replay checks the dispatch and context before folding new identities. An existing-file edit compares the completed writer handle with the observed native file and records `rule_edit` under the same dispatch and Context Epoch checks. A failed read leaves its scoped rule eligible for that edit. Direct patches collect up to 32 completed native effects in execution order. Each delivered rule records its matching physical target and the settlement stores the effect receipts under `rule_patch`. The shared discovery and rendering budgets apply to the whole patch. Creation uses the completed exclusive writer and bound parent. Removal executes relative to the observed parent directory and attests the successful removal of its name and the resulting absence. A later patch failure preserves the error status and retains the completed effects. Bash and PowerShell interceptions reuse the patch receipts and replay requires the interception marker for shell calls. Exact configuration generation and revocation/restart acceptance remain open. Represent rules as typed Context Sources so inspection and model assembly use the same effective data. `GET /api/v1/sessions/{id}/context` and `/context` report Epoch, source identity/origin, estimated tokens largest first, source status, pending changes, totals and utilization with unknown denominators explicitly unknown. Inspection is read-only: it must not start inference, launch integrations or load an untrusted/denied file just to count it. Bound response sizes and omit secret-bearing source content by default. Complete the existing 32 KiB per-file and 128 KiB aggregate instruction-budget semantics through the proposed additive `instruction_limits: { max_file_bytes: 32768, max_total_bytes: 131072 }` configuration. Preserve the `instructions` array type. Update existing target wording that expresses this as `instructions.max_bytes`, since that property cannot coexist on an array. The new setting and its override/managed-policy behavior are part of artifact review. The setting follows normal per-member configuration precedence, including selected profiles and command-line overrides. Project definitions, including those inside profiles, use the existing sensitive-definition trust gate because they can increase host text reads. It does not add a managed-policy namespace or allow project/CLI writes to `policy`; unsupported managed keys keep their existing refusal behavior. Runtime and doctor require the resolved limits before reading instruction files. Nested reminders retain their 32 KiB ceiling and honor a lower configured total. The complete rendered baseline and nested reminder fit their total budgets, including labels, notices, separators, wrappers and bounded file lists. Delivery records use the exact typed set of labelled files, never parsed content. Unlisted imports remain discoverable through bounded authorized re-observation of their parent, and in-group deduplication happens only after successful finalization. Truncated import lines grant no path authority. Path-rule definition/operation/Epoch identity and effective inspection remain required. diff --git a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md index fd9745542..0571ac4da 100644 --- a/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md +++ b/openspec/changes/complete-daily-coding-surfaces/specs/system-context/spec.md @@ -79,7 +79,7 @@ #### Scenario: A patch fails after a completed file change -- **GIVEN** a direct patch has several authorized file changes +- **GIVEN** a built-in patch, including one intercepted from a shell call, has several authorized file changes - **WHEN** the first change completes and a later precondition fails - **THEN** the patch SHALL retain its error status - **AND** rule discovery SHALL consider the verified completed effects under the shared reminder budget diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index ff65b5f75..3371071aa 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -1835,3 +1835,21 @@ for Core, Tools, Server and App. Strict OpenSpec validated all 100 items. Spec lint reported zero errors and 25 warnings. Formatting and generated SDK consistency passed. Windows, full workspace and SDK runtime suites were not rerun locally. Current-source native acceptance remains pending. + +Intercepted shell patch continuation. The built-in patch path records the same +authorized receipts for Bash and PowerShell interceptions. Shell settlement +retains the original dispatch name and the interception metadata. Replay +requires that marker in addition to the attempt, Location, root, operation +target and Context Epoch. A regular external command supplies no built-in +patch receipts. App regressions cover complete and partial Bash interceptions +and reject replay after removing their interception metadata. The shell suite +checks that an edit denial and external program execution cannot supply +receipts. The external program and login shell cases check routing and receipts +because the external program can legitimately change the fixture file. + +Local verification passed 8 shell patch tests, 18 App instruction tests, 455 +Server library tests (one ignored subprocess probe), and all-target Clippy for +Tools, Server and App. Formatting, workflow YAML, strict OpenSpec (100 items) +and SDK consistency passed. Spec lint reported zero errors and 25 warnings. +The PowerShell receipt assertion is Windows-specific and did not run locally. +Full workspace, SDK runtime and current-source platform acceptance remain open. From eb1f298f44f00e3928e7e75220bbc66e5407e351 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 23:09:25 -0400 Subject: [PATCH 83/87] Add authenticated stored context inspection and TUI panel --- crates/cyber-app/tests/instructions.rs | 16 +++ crates/cyber-server/src/http/openapi.rs | 3 +- crates/cyber-server/src/http/sessions.rs | 41 +++++- crates/cyber-server/src/runtime/context.rs | 20 +++ .../src/runtime/context_inspection.rs | 127 ++++++++++++++++++ crates/cyber-server/src/runtime/drain.rs | 14 +- crates/cyber-server/src/runtime/mod.rs | 2 + crates/cyber-server/tests/http.rs | 57 ++++++++ crates/cyber-tui/src/app.rs | 27 +++- crates/cyber-tui/src/context_inspection.rs | 60 +++++++++ crates/cyber-tui/src/lib.rs | 1 + crates/cyber-tui/src/mouse.rs | 1 + crates/cyber-tui/src/perform.rs | 21 +++ crates/cyber-tui/src/runner.rs | 60 +++++++++ crates/cyber-tui/src/view/mod.rs | 3 + crates/cyber-tui/src/view/overlays.rs | 1 + docs/goal-delivery.md | 21 ++- docs/local-core-guide.md | 10 +- .../complete-daily-coding-surfaces/tasks.md | 1 + sdk/openapi.json | 111 ++++++++++++++- sdk/typescript/src/generated/operations.ts | 12 +- sdk/typescript/src/generated/types.ts | 27 ++++ 22 files changed, 604 insertions(+), 32 deletions(-) create mode 100644 crates/cyber-server/src/runtime/context_inspection.rs create mode 100644 crates/cyber-tui/src/context_inspection.rs diff --git a/crates/cyber-app/tests/instructions.rs b/crates/cyber-app/tests/instructions.rs index 6f2e04f49..94b3e30d4 100644 --- a/crates/cyber-app/tests/instructions.rs +++ b/crates/cyber-app/tests/instructions.rs @@ -625,6 +625,14 @@ async fn cyber_md_reaches_the_model_after_agents_md_nested_and_after_edits() { ); assert!(!first.contains("Web cyber rule."), "{first}"); let context = f + .call( + Method::GET, + &format!("/sessions/{session}/context?include_content=true"), + &repo, + None, + ) + .await; + let safe = f .call( Method::GET, &format!("/sessions/{session}/context"), @@ -632,6 +640,14 @@ async fn cyber_md_reaches_the_model_after_agents_md_nested_and_after_edits() { None, ) .await; + assert!(safe["data"]["baseline"].is_null()); + assert!(!safe.to_string().contains("Root cyber rule.")); + assert!( + safe["data"]["inspection"]["estimated_input_tokens"] + .as_u64() + .unwrap() + > 0 + ); let baseline = context["data"]["baseline"].as_str().unwrap(); let source = format!("Instructions from: {}", repo.join("CYBER.md").display()); assert!(baseline.contains(&source), "{baseline}"); diff --git a/crates/cyber-server/src/http/openapi.rs b/crates/cyber-server/src/http/openapi.rs index 0a24868ee..bf523e1cb 100644 --- a/crates/cyber-server/src/http/openapi.rs +++ b/crates/cyber-server/src/http/openapi.rs @@ -947,8 +947,9 @@ pub fn operations(g: &mut SchemaGenerator) -> Vec { "get", "/sessions/{sessionID}/context", "v1.session.context", - "Context baseline, summary and task state", + "Inspect recorded context and estimates. Use include_content to return baseline, summary and task text.", ) + .query(&["include_content"]) .ok::>(g), op( "get", diff --git a/crates/cyber-server/src/http/sessions.rs b/crates/cyber-server/src/http/sessions.rs index 1b3f41e69..0e37af669 100644 --- a/crates/cyber-server/src/http/sessions.rs +++ b/crates/cyber-server/src/http/sessions.rs @@ -79,7 +79,9 @@ pub struct SessionContext { pub epoch: Option, pub baseline: Option, pub summary: Option, + /// Task text is included only on explicit request. pub task: TaskState, + pub inspection: crate::runtime::ContextInspection, } #[derive(Debug, Serialize, JsonSchema)] @@ -1187,18 +1189,45 @@ async fn release( Ok(StatusCode::NO_CONTENT) } +#[derive(Debug, Deserialize)] +struct ContextQuery { + #[serde(default)] + include_content: bool, +} + async fn context( State(state): State, Path(id): Path, + Query(query): Query, ) -> Result>> { let s = state.runtime.state(&id).await?; - let data = SessionContext { - epoch: s.epoch.as_ref().map(|e| e.number), - baseline: s.epoch.map(|e| e.baseline), - summary: s.compacted.map(|c| c.summary), - task: s.task, + let inspection = state.runtime.inspect_context(&s); + let epoch = s.epoch.as_ref().map(|e| e.number); + let (baseline, summary, task) = if query.include_content { + ( + s.epoch.map(|e| e.baseline), + s.compacted.map(|c| c.summary), + s.task, + ) + } else { + ( + None, + None, + TaskState { + version: s.task.version, + ..TaskState::default() + }, + ) }; - Ok(Json(Data { data })) + Ok(Json(Data { + data: SessionContext { + epoch, + baseline, + summary, + task, + inspection, + }, + })) } async fn diff( diff --git a/crates/cyber-server/src/runtime/context.rs b/crates/cyber-server/src/runtime/context.rs index dbd8929f2..f9b019dc4 100644 --- a/crates/cyber-server/src/runtime/context.rs +++ b/crates/cyber-server/src/runtime/context.rs @@ -70,6 +70,26 @@ pub fn snapshot( .collect()) } +/// Persisted system text shared by request assembly and context inspection. +pub(super) fn request_system(epoch: &super::model::Epoch, family: ModelFamily) -> [String; 2] { + [ + epoch + .system_prefix + .clone() + .unwrap_or_else(|| base_prompt(family)), + epoch.baseline.clone(), + ] +} + +pub(super) fn epoch_pending(state: &super::SessionState, model: &super::ResolvedModel) -> bool { + let version = cyber_core::prompts::base_prompt_version(); + state.epoch_stale + || state + .epoch + .as_ref() + .is_none_or(|epoch| !epoch.serves(&model.provider, model.family.as_str(), version)) +} + /// The snapshot of a new epoch. After the first one an unavailable source keeps its /// previous value (or stays absent), so a transient read failure never blocks a Turn; /// only the first epoch is blocked by one. diff --git a/crates/cyber-server/src/runtime/context_inspection.rs b/crates/cyber-server/src/runtime/context_inspection.rs new file mode 100644 index 000000000..ea3b2c096 --- /dev/null +++ b/crates/cyber-server/src/runtime/context_inspection.rs @@ -0,0 +1,127 @@ +//! Read-only inspection of the Session state used by request assembly. + +use cyber_llm::Message; +use schemars::JsonSchema; +use serde::Serialize; + +use super::{InputStatus, Runtime, SessionState, view}; + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ContextSourceEstimate { + pub id: String, + pub origin: String, + pub estimated_tokens: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ContextInspection { + /// Sources in the stored projection, ordered by estimated size. + pub sources: Vec, + /// Estimate of the epoch system text and projected messages. Tool schemas + /// and instructions added only while assembling a turn are excluded. + pub estimated_input_tokens: Option, + pub window: Option, + pub estimated_input_utilization: Option, + /// Measured prompt tokens of the last finished request, when available. + pub last_request_tokens: Option, + /// The next turn needs to rebuild the epoch. + pub epoch_pending: bool, + pub queued_inputs: usize, + pub held_inputs: usize, + pub delivered_rule_count: usize, +} + +impl Runtime { + pub fn inspect_context(&self, state: &SessionState) -> ContextInspection { + let model = self.inner.resolve(&state.info.model).ok(); + let window = model + .as_ref() + .and_then(|model| (model.context_limit > 0).then_some(model.context_limit)); + let mut inspection = ContextInspection { + sources: Vec::new(), + estimated_input_tokens: None, + window, + estimated_input_utilization: None, + last_request_tokens: state + .last_step + .as_ref() + .map(|step| step.usage.context_tokens()), + epoch_pending: model + .as_ref() + .map_or(state.epoch_stale || state.epoch.is_none(), |model| { + super::context::epoch_pending(state, model) + }), + queued_inputs: count_inputs(state, InputStatus::Pending), + held_inputs: count_inputs(state, InputStatus::Held), + delivered_rule_count: state + .epoch + .as_ref() + .map_or(0, |epoch| epoch.reminded_rules.len()), + }; + let (Some(epoch), Some(model)) = (&state.epoch, model) else { + return inspection; + }; + let system = super::context::request_system(epoch, model.family); + let messages = view::messages(state, &view::Replay::of(&model)); + inspection.sources = estimates(&system, &messages, state.compacted.is_some()); + let estimate = view::estimate_tokens(&system, &messages, &[]); + inspection.estimated_input_tokens = Some(estimate); + inspection.estimated_input_utilization = + window.map(|window| estimate as f64 / window as f64); + inspection + } +} + +fn count_inputs(state: &SessionState, status: InputStatus) -> usize { + state + .inbox + .iter() + .filter(|input| input.status == status) + .count() +} + +fn estimates( + system: &[String; 2], + messages: &[Message], + compacted: bool, +) -> Vec { + let mut sources = vec![ + source( + "system_prefix", + "context_epoch", + (system[0].len() / 4) as u64, + ), + source("baseline", "context_epoch", (system[1].len() / 4) as u64), + ]; + let history = if compacted && let Some((summary, tail)) = messages.split_first() { + sources.push(source( + "summary_and_task", + "compaction", + view::estimate_tokens(&[], std::slice::from_ref(summary), &[]), + )); + tail + } else { + messages + }; + if !history.is_empty() { + sources.push(source( + "history", + "session_replay", + view::estimate_tokens(&[], history, &[]), + )); + } + sources.sort_by(|a, b| { + b.estimated_tokens + .cmp(&a.estimated_tokens) + .then(a.id.cmp(&b.id)) + }); + sources +} + +fn source(id: &str, origin: &str, estimated_tokens: u64) -> ContextSourceEstimate { + ContextSourceEstimate { + id: id.into(), + origin: origin.into(), + estimated_tokens, + } +} diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index f9e695eec..40e454cea 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -816,11 +816,7 @@ impl Inner { ) -> Result<(), RuntimeError> { { let state = handle.state.lock().await; - let version = cyber_core::prompts::base_prompt_version(); - if let Some(epoch) = &state.epoch - && !state.epoch_stale - && epoch.serves(&resolved.provider, resolved.family.as_str(), version) - { + if !super::context::epoch_pending(&state, resolved) { return Ok(()); } } @@ -1214,13 +1210,7 @@ impl Inner { .as_ref() .ok_or_else(|| RuntimeError::Corrupt("no context epoch".into()))?; let mut request = resolved.template.clone(); - request.system = vec![ - epoch - .system_prefix - .clone() - .unwrap_or_else(|| context::base_prompt(resolved.family)), - epoch.baseline.clone(), - ]; + request.system = context::request_system(epoch, resolved.family).into(); // Only this request: the epoch's system text stays byte-stable. if finalizer.is_some() && root { request diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index c746c948e..0dc349add 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -36,6 +36,7 @@ mod compaction; pub use child_usage::ChildrenUsage; pub mod computer; mod context; +mod context_inspection; mod context_meter; mod continuation; mod delegations; @@ -245,6 +246,7 @@ pub use auto::{AutoDecision, AutoEffect, AutoReview, AutoReviewStage, TAINTED_RE pub use bus::LiveEvent; pub use compaction::CompactionConfig; pub use context::{ContextInputs, Observed as ContextObservation, base_prompt}; +pub use context_inspection::ContextInspection; pub use context_meter::ContextMeter; pub use cyber_core::prompts::ModelFamily; pub use events::{AuxiliaryUsage, CompactionTrigger, registry as event_registry}; diff --git a/crates/cyber-server/tests/http.rs b/crates/cyber-server/tests/http.rs index 03b710685..058944eca 100644 --- a/crates/cyber-server/tests/http.rs +++ b/crates/cyber-server/tests/http.rs @@ -4054,6 +4054,63 @@ async fn context_meter_reports_last_request_prompt_tokens_window_and_compaction_ assert_eq!(session["data"]["context"], expected); } +#[tokio::test] +async fn context_inspection_is_authenticated_read_only_and_metadata_only_by_default() { + let h = Harness::new(Setup { + scripts: vec![("test/main", vec![text("hello")])], + ..Setup::default() + }); + let api = Api::new(&h); + let id = h.session().await; + let base = tcp(&h, "pw").await; + let url = format!("{base}/sessions/{id}/context"); + let client = reqwest::Client::new(); + assert_eq!(client.get(&url).send().await.unwrap().status(), 401); + let inspect = || client.get(&url).basic_auth("cyber", Some("pw")); + let fresh: Value = inspect().send().await.unwrap().json().await.unwrap(); + assert_eq!(fresh["data"]["inspection"]["epoch_pending"], true); + assert_eq!( + fresh["data"]["inspection"]["estimated_input_tokens"], + Value::Null + ); + assert!(h.models.requests("test/main").is_empty()); + + api.post(&format!("/sessions/{id}/prompt"), prompt("hi")) + .await; + h.settle(&id).await; + let result: Value = inspect().send().await.unwrap().json().await.unwrap(); + let data = &result["data"]; + assert!(data["baseline"].is_null()); + assert!(data["summary"].is_null()); + let estimate = &data["inspection"]; + assert_eq!(estimate["epoch_pending"], false); + assert_eq!(estimate["last_request_tokens"], 100); + assert_eq!(estimate["window"], 200_000); + let tokens = estimate["estimated_input_tokens"].as_u64().unwrap(); + assert!(tokens > 0); + assert_eq!( + estimate["estimated_input_utilization"], + tokens as f64 / 200_000.0 + ); + let sources = estimate["sources"].as_array().unwrap(); + assert!(sources.iter().any(|source| source["id"] == "history")); + assert!(sources.windows(2).all(|pair| { + pair[0]["estimated_tokens"].as_u64().unwrap() + >= pair[1]["estimated_tokens"].as_u64().unwrap() + })); + let explicit: Value = inspect() + .query(&[("include_content", true)]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(explicit["data"]["baseline"].is_string()); + assert_eq!(explicit["data"]["inspection"], *estimate); + assert_eq!(h.models.requests("test/main").len(), 1); +} + impl Api { /// Start an inline run owned by `session` and wait until it ends. async fn ended_run(&self, session: &str, script: &str) -> String { diff --git a/crates/cyber-tui/src/app.rs b/crates/cyber-tui/src/app.rs index 02bc31586..3639d3834 100644 --- a/crates/cyber-tui/src/app.rs +++ b/crates/cyber-tui/src/app.rs @@ -168,6 +168,9 @@ pub enum Action { LoadModels, /// `/skills`: the Location's skills for the Session agent, with diagnostics. LoadSkills, + InspectContext { + request_id: String, + }, /// Refetch `/commands`, as MCP prompt commands come and go with connections. LoadCommands, /// `/plugins` (`reload: false`) or `/reload-plugins`: the Location's plugins. @@ -357,6 +360,12 @@ pub enum Overlay { text: String, scroll: u16, }, + /// `/context`: estimates and source metadata for the current Session. + Context { + request_id: String, + text: String, + scroll: u16, + }, /// `/btw`: a side question and its answer. SideChat(crate::side_chat::SideChat), /// `/messages`: the Session's message history; its state is `App::messaging`. @@ -1055,7 +1064,7 @@ impl App { self.file_diff_key(key); Vec::new() } - Overlay::TaskLog { .. } => { + Overlay::TaskLog { .. } | Overlay::Context { .. } => { self.task_log_key(key); Vec::new() } @@ -1190,7 +1199,8 @@ impl App { /// Keys over a task's log. fn task_log_key(&mut self, key: KeyEvent) { - let Overlay::TaskLog { scroll, .. } = &mut self.overlay else { + let (Overlay::TaskLog { scroll, .. } | Overlay::Context { scroll, .. }) = &mut self.overlay + else { return; }; match key.code { @@ -1830,6 +1840,19 @@ impl App { self.toast("Use /hooks to review definitions or /hooks history for receipts"); Vec::new() } + "context" if args.is_empty() => { + let request_id = cyber_core::ids::new_id("ctx"); + self.overlay = Overlay::Context { + request_id: request_id.clone(), + text: "Loading context…".into(), + scroll: 0, + }; + vec![Action::InspectContext { request_id }] + } + "context" => { + self.toast("Use /context without arguments"); + Vec::new() + } "cost" | "usage" => self.open_usage(), "settings" => self.open_settings(None), "status" => self.open_settings(Some(crate::settings::Tab::Status)), diff --git a/crates/cyber-tui/src/context_inspection.rs b/crates/cyber-tui/src/context_inspection.rs new file mode 100644 index 000000000..2864e6855 --- /dev/null +++ b/crates/cyber-tui/src/context_inspection.rs @@ -0,0 +1,60 @@ +//! Format the metadata-only Session context response for the scrollable panel. + +use serde_json::Value; + +pub(crate) fn format(data: &Value) -> String { + let inspection = &data["inspection"]; + if !inspection.is_object() { + return "Context estimates are unavailable from this server.".into(); + } + let count = |value: &Value| { + value + .as_u64() + .map_or_else(|| "unknown".into(), |n| n.to_string()) + }; + let mut lines = vec![ + format!("Context Epoch {}", count(&data["epoch"])), + format!( + "Estimated stored input: {} tokens", + count(&inspection["estimated_input_tokens"]) + ), + format!("Model window: {} tokens", count(&inspection["window"])), + format!( + "Last measured request: {} tokens", + count(&inspection["last_request_tokens"]) + ), + format!( + "Queued inputs: {} · Held inputs: {}", + count(&inspection["queued_inputs"]), + count(&inspection["held_inputs"]) + ), + format!( + "Delivered scoped rules: {}", + count(&inspection["delivered_rule_count"]) + ), + ]; + let utilization = inspection["estimated_input_utilization"] + .as_f64() + .map_or_else( + || "unknown".into(), + |fraction| format!("{:.1}%", fraction * 100.0), + ); + lines.push(format!("Estimated window use: {utilization}")); + if inspection["epoch_pending"].as_bool() == Some(true) { + lines.push("The next turn will rebuild the Context Epoch.".into()); + } + lines.push("Estimate excludes tool schemas and turn-only instructions.".into()); + if let Some(sources) = inspection["sources"].as_array() { + lines.push(String::new()); + for source in sources { + if let (Some(id), Some(origin), Some(tokens)) = ( + source["id"].as_str(), + source["origin"].as_str(), + source["estimated_tokens"].as_u64(), + ) { + lines.push(format!("{id} · {origin} · ~{tokens} tokens")); + } + } + } + lines.join("\n") +} diff --git a/crates/cyber-tui/src/lib.rs b/crates/cyber-tui/src/lib.rs index 2e6b5b1dd..525077561 100644 --- a/crates/cyber-tui/src/lib.rs +++ b/crates/cyber-tui/src/lib.rs @@ -10,6 +10,7 @@ mod children; mod collaboration; mod composer; mod computer; +mod context_inspection; mod explorer; mod export; mod fuzzy; diff --git a/crates/cyber-tui/src/mouse.rs b/crates/cyber-tui/src/mouse.rs index 3a9158c41..a0cf79c74 100644 --- a/crates/cyber-tui/src/mouse.rs +++ b/crates/cyber-tui/src/mouse.rs @@ -698,6 +698,7 @@ fn wheel_moves(overlay: &Overlay) -> bool { | Overlay::Themes(_) | Overlay::Workflows | Overlay::TaskLog { .. } + | Overlay::Context { .. } | Overlay::Messages | Overlay::Team | Overlay::Collaboration diff --git a/crates/cyber-tui/src/perform.rs b/crates/cyber-tui/src/perform.rs index 8e7dbdc5a..8edda3714 100644 --- a/crates/cyber-tui/src/perform.rs +++ b/crates/cyber-tui/src/perform.rs @@ -184,6 +184,12 @@ pub enum Msg { Sessions(Vec), Catalog(crate::model::Catalog), Models(Vec), + Context { + session_id: String, + directory: String, + request_id: String, + result: Result, + }, Skills { directory: String, result: Result, String>, @@ -976,6 +982,7 @@ pub async fn perform_owned( | Action::Delete(_) | Action::ForkSession(_) => manage(client, action).await, Action::LoadModels + | Action::InspectContext { .. } | Action::LoadSkills | Action::LoadCommands | Action::LoadPlugins { .. } @@ -1085,6 +1092,20 @@ async fn inspect_hooks(client: &Client, session: &Session, action: Action) -> Re async fn load_choices(client: &Client, session: &Session, action: Action) -> Result { match action { + Action::InspectContext { request_id } => { + let path = format!("/sessions/{}/context", encode(&session.id)); + let result = client + .get(&path) + .await + .map(|body| crate::context_inspection::format(&body["data"])) + .map_err(err); + Ok(Msg::Context { + session_id: session.id.clone(), + directory: session.directory.clone(), + request_id, + result, + }) + } Action::LoadModels => models(client).await, Action::LoadSkills => { let path = format!("/skills?agent={}", encode(&session.agent)); diff --git a/crates/cyber-tui/src/runner.rs b/crates/cyber-tui/src/runner.rs index 05ef00cbf..bd3043872 100644 --- a/crates/cyber-tui/src/runner.rs +++ b/crates/cyber-tui/src/runner.rs @@ -803,6 +803,24 @@ fn apply_msg(app: &mut App, msg: Result, refresh: &mut Refresh) -> } Msg::Sessions(items) => app.open_picker(PickerKind::Sessions, "Sessions", items), Msg::Models(items) => app.open_picker(PickerKind::Models, "Models", items), + Msg::Context { + session_id, + directory, + request_id, + result, + } => { + if session_id == app.session.id + && directory == app.session.directory + && let crate::app::Overlay::Context { + request_id: pending, + text, + .. + } = &mut app.overlay + && *pending == request_id + { + *text = result.unwrap_or_else(|error| format!("Context unavailable: {error}")); + } + } Msg::Skills { directory, result } => open_skills(app, &directory, result), Msg::Plugins { directory, @@ -1633,6 +1651,48 @@ mod mode_tests { assert!(app.agents.is_empty()); } + #[test] + fn context_command_only_applies_the_current_inspection_reply() { + use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; + let mut app = app(); + for c in "/context".chars() { + app.on_key(KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)); + } + let actions = app.on_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + let [Action::InspectContext { request_id }] = actions.as_slice() else { + panic!("expected one context inspection action, got {actions:?}"); + }; + let overlay = app.overlay.clone(); + let session_id = app.session.id.clone(); + let directory = app.session.directory.clone(); + let mut refresh = Refresh::default(); + for (session, location, request) in [ + ("ses_old", directory.as_str(), request_id.as_str()), + (session_id.as_str(), "/old", request_id.as_str()), + (session_id.as_str(), directory.as_str(), "ctx_old"), + ] { + let message = Msg::Context { + session_id: session.into(), + directory: location.into(), + request_id: request.into(), + result: Ok("stale reply".into()), + }; + assert!(apply(&mut app, Ok(message), &mut refresh).is_empty()); + assert_eq!(app.overlay, overlay); + } + let message = Msg::Context { + session_id, + directory, + request_id: request_id.clone(), + result: Ok("current reply".into()), + }; + assert!(apply(&mut app, Ok(message), &mut refresh).is_empty()); + assert!(matches!( + app.overlay, + crate::app::Overlay::Context { text, .. } if text == "current reply" + )); + } + fn skill_catalog_choices() -> Vec { let catalog = serde_json::json!({"agent":"build","skills":[ {"name":"deploy","description":"Deploy","source":"/repo/.cyber/skills/deploy","scope":"project", diff --git a/crates/cyber-tui/src/view/mod.rs b/crates/cyber-tui/src/view/mod.rs index c5b9ccc91..2874e1929 100644 --- a/crates/cyber-tui/src/view/mod.rs +++ b/crates/cyber-tui/src/view/mod.rs @@ -163,6 +163,9 @@ pub fn draw(f: &mut Frame, app: &App) { text, scroll, } => overlays::task_log(f, app, title, text, *scroll, area), + Overlay::Context { text, scroll, .. } => { + overlays::task_log(f, app, "Context", text, *scroll, area) + } Overlay::SideChat(chat) => overlays::side_chat(f, app, chat, area), Overlay::ConfirmStopTasks => overlays::confirm_stop_tasks(f, app, area), Overlay::ConfirmExport(request) => overlays::confirm_export(f, app, request, area), diff --git a/crates/cyber-tui/src/view/overlays.rs b/crates/cyber-tui/src/view/overlays.rs index 456d48f34..bd67d1398 100644 --- a/crates/cyber-tui/src/view/overlays.rs +++ b/crates/cyber-tui/src/view/overlays.rs @@ -592,6 +592,7 @@ pub(crate) const HELP: &[(&str, &str)] = &[ "/memory [global]", "review notes, deletion and pinned recovery", ), + ("/context", "inspect the current Session context"), ("/cost", "token classes, subtree cost and cache hit rate"), ( "/hooks history", diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index f66128dc3..3fd3b6115 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,12 +21,31 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads, edits and patches, including intercepted shell patches, use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native operation and dispatched attempt. A partial patch can deliver rules for completed files. Finish exact config-generation and revocation/restart provenance. Extend the existing `/sessions/:id/context` endpoint beyond epoch/baseline/summary/task to effective source inventory, estimates/status/pending changes; add the owner-facing `/context` flow and leakage/restart/revocation acceptance. The sidebar meter is not that inspection feature. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads, edits and patches, including intercepted shell patches, use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native operation and dispatched attempt. A partial patch can deliver rules for completed files. Finish exact config-generation and revocation/restart provenance. The authenticated Session context route and TUI `/context` now show stored source groups, projected token estimates, measured usage and pending Epoch state. The default response hides stored text. Finish per-source provenance, revocation and pending source changes, and complete leakage, restart and native acceptance. | | #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | | #92 protocol/callbacks/producers, tasks 3–6/S/D | Finish contribution ownership, hooks/providers/catalog transforms and TypeScript kit, durable scoped callbacks (`session/prompt` still explicitly refuses), Setup and DirectoryAdded plus combined producer acceptance. Preserve accepted FileChanged and CwdChanged fixes. Complete S1 epoch/directory-key/legacy-deletion and D1 directory-admission/Safe Boundary/child-policy review before their dependent code. | +### Stored context inspection continuation + +The authenticated context route now adds source-group estimates from the shared +persisted system text and replay projection. Default responses omit baseline, +summary and task text. Explicit `include_content=true` restores those fields. +The TUI `/context` panel fences replies by Session, Location and request identity. +Epoch rebuild status uses the runtime's provider/family/prompt-version predicate. +These estimates exclude tool schemas and turn-only additions. Individual source +provenance and pending source changes remain required by task 4.3. + +Local verification passed 18 application instruction tests, 13 server context +unit tests, two HTTP context tests, ten TUI context tests, the OpenAPI artifact +check, SDK typecheck and 136 SDK tests (ten configured skips). Package library +Clippy passed after routing inspection through the existing loader. Strict +OpenSpec validation passed 100 items and lint reported zero errors/25 warnings. +Native macOS C deprecation and linker unwind-size warnings were observed. +Source-bound native CI remains pending on preceding revision `a4d28df4`. +This continuation has no native/platform acceptance. + ### Zero-byte ordinary restore continuation Ordinary cleanup now records payload completion for exact empty inventories and diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 444489c76..29d85c56e 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1223,7 +1223,13 @@ Definition reads use the fenced native reader: linked files/directories, hard li Scoped rules activate after a successful built-in read, edit, or file change from `apply_patch` with a native receipt. Rule matching uses the physical file from the completed operation after authorization and any hook rewrite. Writes are bound to the handle that completed the effect. Patch deletions bind the physical parent directory and removed name. A partially applied patch keeps its error status and can deliver rules for its verified completed files. A directory listing, unsupported observation, or changed root/target binding does not consume delivery. The tool's own result is preserved. Ordinary instruction behavior remains available to legacy hosts without native receipts; those hosts cannot activate path-scoped rules. -The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Patch rule discovery shares the reminder budget across up to 32 verified file effects. Patches intercepted by the built-in Bash or PowerShell path use the same receipts. Shell settlements record the interception and replay requires it. Exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and effective `/context` inspection remain in progress. +The runtime resolves settings after the read and verifies its captured Location, project root, file binding and Context Epoch before discovery and settlement. A delivered rule records its native root, definition path, digest and approval generation. The settlement also records the actual read's file identity, call attempt, operation key, Location and Epoch. Historical path-only reminders still replay. Patch rule discovery shares the reminder budget across up to 32 verified file effects. Patches intercepted by the built-in Bash or PowerShell path use the same receipts. Shell settlements record the interception and replay requires it. Exact configuration-generation fencing through commit, withdrawal after revocation, full restart acceptance and per-source provenance in `/context` remain in progress. + +### Inspecting the current context + +Open `/context` in the TUI to see the stored Context Epoch, projected source groups, estimated token counts and the last measured request. The estimate uses the persisted system text and projected conversation. Tool schemas and instructions added only during a turn are excluded. The panel also reports whether an Epoch rebuild is pending, how many inputs are queued or held, and how many scoped rules were delivered in the Epoch. + +Authenticated clients can use `GET /api/v1/sessions/{sessionID}/context` for the same metadata. Pass `include_content=true` to retrieve the stored baseline, compaction summary and task text. Inspection reads stored Session state without starting a turn or reloading instruction files. ### Switches @@ -1236,7 +1242,7 @@ These are read from the server's environment (a value of `1`, `true`, `yes` or ` ### Checking what loads -`cyber doctor` has an `instructions` line that lists, in load order, the files a new Session at the current directory loads (`N loaded: , ...`), and warns about truncated files and unresolved or refused imports. `GET /api/v1/sessions/{id}/context` returns the rendered baseline with its `Instructions from:` labels. +`cyber doctor` has an `instructions` line that lists, in load order, the files a new Session at the current directory loads (`N loaded: , ...`), and warns about truncated files and unresolved or refused imports. `GET /api/v1/sessions/{id}/context?include_content=true` returns the rendered baseline with its `Instructions from:` labels. ### Migrating from Claude Code diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 3371071aa..47c1905af 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -130,6 +130,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - [ ] 4.1 Implement trusted path-scoped rule discovery/precedence, validated globs, overrides and bounded instruction assembly without changing `instructions` array compatibility. - [ ] 4.2 Bind reminder delivery to successful authorized operations and Context Epoch identity, including edits/patches, partial settlements, compaction/restart and changed/revoked rules. - [ ] 4.3 Add authenticated read-only context inspection and `/context`, source estimates/origins/status/pending updates and honest utilization; synchronize OpenAPI/SDK and doctor diagnostics. + - The Session endpoint and TUI panel expose stored source groups and an estimate based on request replay. The default response omits stored text. Individual source provenance, pending source changes and next-request estimates remain open. - [ ] 4.4 Test no rule leakage across Location/Epoch, denied/failed calls, truncation, untrusted sources, stale response and token-count consistency with actual request assembly. ## 5. Root structured output diff --git a/sdk/openapi.json b/sdk/openapi.json index 47967ac34..188a229a4 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -9695,11 +9695,16 @@ ] }, "task": { - "$ref": "#/components/schemas/TaskState" + "$ref": "#/components/schemas/TaskState", + "description": "Task text is included only on explicit request." + }, + "inspection": { + "$ref": "#/components/schemas/ContextInspection" } }, "required": [ - "task" + "task", + "inspection" ] }, "TaskState": { @@ -9750,6 +9755,98 @@ ], "description": "A user statement with its source, kept outside generated prose." }, + "ContextInspection": { + "type": "object", + "properties": { + "sources": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ContextSourceEstimate" + }, + "description": "Sources in the stored projection, ordered by estimated size." + }, + "estimated_input_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0, + "description": "Estimate of the epoch system text and projected messages. Tool schemas\nand instructions added only while assembling a turn are excluded." + }, + "window": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "estimated_input_utilization": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "last_request_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0, + "description": "Measured prompt tokens of the last finished request, when available." + }, + "epoch_pending": { + "type": "boolean", + "description": "The next turn needs to rebuild the epoch." + }, + "queued_inputs": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "held_inputs": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "delivered_rule_count": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "sources", + "epoch_pending", + "queued_inputs", + "held_inputs", + "delivered_rule_count" + ] + }, + "ContextSourceEstimate": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "origin": { + "type": "string" + }, + "estimated_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "id", + "origin", + "estimated_tokens" + ] + }, "Data_Array_of_FileDiff": { "type": "object", "properties": { @@ -26714,7 +26811,7 @@ "/api/v1/sessions/{sessionID}/context": { "get": { "operationId": "v1.session.context", - "summary": "Context baseline, summary and task state", + "summary": "Inspect recorded context and estimates. Use include_content to return baseline, summary and task text.", "tags": [ "session" ], @@ -26726,6 +26823,14 @@ "schema": { "type": "string" } + }, + { + "name": "include_content", + "in": "query", + "required": false, + "schema": { + "type": "string" + } } ], "responses": { diff --git a/sdk/typescript/src/generated/operations.ts b/sdk/typescript/src/generated/operations.ts index e66c4151a..0b08b323e 100644 --- a/sdk/typescript/src/generated/operations.ts +++ b/sdk/typescript/src/generated/operations.ts @@ -1211,7 +1211,9 @@ export interface Operations { }; "v1.session.context": { path: { sessionID: string }; - query: Record; + query: { + include_content?: QueryValue; + }; body: undefined; response: S.SessionContext; }; @@ -4084,7 +4086,7 @@ export const operations = { method: "GET", path: "/api/v1/sessions/{sessionID}/context", pathParams: ["sessionID"], - query: [], + query: ["include_content"], body: "none", located: false, unwrap: true, @@ -6113,9 +6115,9 @@ export function createGroups(caller: Caller) { /** Revoke this Session's native grant (`POST /api/v1/sessions/{sessionID}/computer/stop`) */ computerStop: (sessionID: string, options?: RequestOptions): Promise => caller.call("v1.session.computerStop", { path: { sessionID } }, options), - /** Context baseline, summary and task state (`GET /api/v1/sessions/{sessionID}/context`) */ - context: (sessionID: string, options?: RequestOptions): Promise => - caller.call("v1.session.context", { path: { sessionID } }, options), + /** Inspect recorded context and estimates. Use include_content to return baseline, summary and task text. (`GET /api/v1/sessions/{sessionID}/context`) */ + context: (sessionID: string, query?: Operations["v1.session.context"]["query"], options?: RequestOptions): Promise => + caller.call("v1.session.context", { path: { sessionID }, query }, options), /** Create a Session (`POST /api/v1/sessions`) */ create: (body?: Operations["v1.session.create"]["body"], options?: RequestOptions): Promise => caller.call("v1.session.create", { body }, options), diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index cdc6d91ab..230cbf600 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -736,6 +736,25 @@ export type Content2 = { truncated: boolean; }; +export type ContextInspection = { + /** Sources in the stored projection, ordered by estimated size. */ + sources: ContextSourceEstimate[]; + /** + * Estimate of the epoch system text and projected messages. Tool schemas + * and instructions added only while assembling a turn are excluded. + */ + estimated_input_tokens?: number | null; + window?: number | null; + estimated_input_utilization?: number | null; + /** Measured prompt tokens of the last finished request, when available. */ + last_request_tokens?: number | null; + /** The next turn needs to rebuild the epoch. */ + epoch_pending: boolean; + queued_inputs: number; + held_inputs: number; + delivered_rule_count: number; +}; + export type ContextMeter = { /** Prompt tokens of the last request: input + cache read + cache write. */ used_tokens: number; @@ -757,6 +776,12 @@ export type ContextMeter = { compacted: boolean; }; +export type ContextSourceEstimate = { + id: string; + origin: string; + estimated_tokens: number; +}; + export type Contribution = { label: string; commit: string; @@ -4464,7 +4489,9 @@ export type SessionContext = { epoch?: number | null; baseline?: string | null; summary?: string | null; + /** Task text is included only on explicit request. */ task: TaskState; + inspection: ContextInspection; }; /** One exported Session. */ From cf016df09f489f63593b12e95b0e0706e3580448 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 23:18:29 -0400 Subject: [PATCH 84/87] Persist context source provenance and observation status --- .github/workflows/non-git-snapshot-proof.yml | 6 +- crates/cyber-server/src/runtime/context.rs | 14 ++- .../src/runtime/context_inspection.rs | 50 ++++++++ .../src/runtime/context_sources.rs | 113 ++++++++++++++++++ crates/cyber-server/src/runtime/drain.rs | 40 ++++++- crates/cyber-server/src/runtime/events.rs | 3 + crates/cyber-server/src/runtime/mod.rs | 2 + crates/cyber-server/src/runtime/model.rs | 11 ++ .../cyber-server/tests/context_compaction.rs | 110 +++++++++++++++++ crates/cyber-server/tests/http.rs | 12 +- crates/cyber-tui/src/context_inspection.rs | 65 ++++++++++ docs/goal-delivery.md | 26 +++- docs/local-core-guide.md | 2 +- .../complete-daily-coding-surfaces/tasks.md | 2 +- sdk/openapi.json | 76 ++++++++++++ sdk/typescript/src/generated/types.ts | 15 +++ 16 files changed, 539 insertions(+), 8 deletions(-) create mode 100644 crates/cyber-server/src/runtime/context_sources.rs diff --git a/.github/workflows/non-git-snapshot-proof.yml b/.github/workflows/non-git-snapshot-proof.yml index 7f4d87840..f68ba3cf3 100644 --- a/.github/workflows/non-git-snapshot-proof.yml +++ b/.github/workflows/non-git-snapshot-proof.yml @@ -41,11 +41,13 @@ on: - crates/cyber-tools/tests/instruction_config.rs - crates/cyber-tools/tests/path_skills.rs - crates/cyber-tools/tests/shell_patch.rs - - crates/cyber-server/src/runtime/context.rs + - crates/cyber-server/src/runtime/context*.rs - crates/cyber-server/src/runtime/instructions.rs - crates/cyber-server/src/runtime/instructions/** - crates/cyber-server/src/runtime/host.rs - crates/cyber-server/tests/context_compaction.rs + - crates/cyber-server/tests/http.rs + - crates/cyber-tui/src/context_inspection.rs - crates/cyber-server/tests/support/mod.rs - crates/cyber-cli/Cargo.toml - crates/cyber-cli/src/** @@ -210,8 +212,10 @@ jobs: cargo test --locked -p cyber-core --test config rule_definition_bytes -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::context:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::instructions:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-server --lib runtime::context_sources:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --lib runtime::drain::skill_reminder_tests:: -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-server --test context_compaction -- --nocapture 2>&1 | tee -a instruction-limits-proof.log + cargo test --locked -p cyber-server --test http context_inspection -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-app --features native-git-acceptance --test instructions -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-tools --test instruction_config -- --nocapture 2>&1 | tee -a instruction-limits-proof.log cargo test --locked -p cyber-tools --test path_skills -- --nocapture 2>&1 | tee -a instruction-limits-proof.log diff --git a/crates/cyber-server/src/runtime/context.rs b/crates/cyber-server/src/runtime/context.rs index f9b019dc4..16d75b8a5 100644 --- a/crates/cyber-server/src/runtime/context.rs +++ b/crates/cyber-server/src/runtime/context.rs @@ -32,6 +32,8 @@ pub enum Observed { #[derive(Debug, Default)] pub struct Observation { pub sources: BTreeMap, + pub origins: BTreeMap, + pub pending: std::collections::BTreeSet, /// Recorded while reading, never parsed back out of rendered (untrusted) text. pub files: BTreeMap>, } @@ -46,7 +48,17 @@ pub fn observe(inputs: &ContextInputs) -> Observation { let (instructions, read) = instructions(inputs); sources.insert("core/instructions".to_string(), instructions); let files = BTreeMap::from([("core/instructions".to_string(), read)]); - Observation { sources, files } + let origins = BTreeMap::from([ + ("core/date".into(), "runtime".into()), + ("core/environment".into(), "runtime".into()), + ("core/instructions".into(), "instruction_files".into()), + ]); + Observation { + sources, + origins, + files, + pending: Default::default(), + } } /// Values of available sources, or the keys of unavailable ones. diff --git a/crates/cyber-server/src/runtime/context_inspection.rs b/crates/cyber-server/src/runtime/context_inspection.rs index ea3b2c096..1a8b346e5 100644 --- a/crates/cyber-server/src/runtime/context_inspection.rs +++ b/crates/cyber-server/src/runtime/context_inspection.rs @@ -4,8 +4,16 @@ use cyber_llm::Message; use schemars::JsonSchema; use serde::Serialize; +use super::context_sources::{SourceMetadata, SourceStatus}; use super::{InputStatus, Runtime, SessionState, view}; +#[derive(Debug, Serialize, JsonSchema)] +pub struct EffectiveContextSource { + pub id: String, + #[serde(flatten)] + pub metadata: SourceMetadata, +} + #[derive(Debug, Serialize, JsonSchema)] pub struct ContextSourceEstimate { pub id: String, @@ -15,6 +23,9 @@ pub struct ContextSourceEstimate { #[derive(Debug, Serialize, JsonSchema)] pub struct ContextInspection { + /// Latest recorded source observations, without source content. + pub effective_sources: Vec, + pub omitted_sources: usize, /// Sources in the stored projection, ordered by estimated size. pub sources: Vec, /// Estimate of the epoch system text and projected messages. Tool schemas @@ -38,6 +49,8 @@ impl Runtime { .as_ref() .and_then(|model| (model.context_limit > 0).then_some(model.context_limit)); let mut inspection = ContextInspection { + effective_sources: Vec::new(), + omitted_sources: 0, sources: Vec::new(), estimated_input_tokens: None, window, @@ -58,6 +71,43 @@ impl Runtime { .as_ref() .map_or(0, |epoch| epoch.reminded_rules.len()), }; + if let Some(epoch) = &state.epoch { + let ids: std::collections::BTreeSet<_> = + epoch.snapshot.keys().chain(epoch.sources.keys()).collect(); + inspection.omitted_sources = ids.len().saturating_sub(256); + inspection.effective_sources = ids + .into_iter() + .map(|id| { + let metadata = + epoch + .sources + .get(id) + .cloned() + .unwrap_or_else(|| SourceMetadata { + origin: "unknown".into(), + status: SourceStatus::Unknown, + pending_update: false, + estimated_tokens: epoch + .snapshot + .get(id) + .map(|text| (text.len() / 4) as u64), + file_paths: Vec::new(), + omitted_file_paths: 0, + }); + EffectiveContextSource { + id: id.clone(), + metadata, + } + }) + .collect(); + inspection.effective_sources.sort_by(|a, b| { + b.metadata + .estimated_tokens + .cmp(&a.metadata.estimated_tokens) + .then(a.id.cmp(&b.id)) + }); + inspection.effective_sources.truncate(256); + } let (Some(epoch), Some(model)) = (&state.epoch, model) else { return inspection; }; diff --git a/crates/cyber-server/src/runtime/context_sources.rs b/crates/cyber-server/src/runtime/context_sources.rs new file mode 100644 index 000000000..6db2d5b39 --- /dev/null +++ b/crates/cyber-server/src/runtime/context_sources.rs @@ -0,0 +1,113 @@ +//! Metadata captured while context is observed, without retaining duplicate text. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::context::{Observation, Observed}; + +pub const OBSERVED: &str = "session.context.observed.1"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SourceStatus { + /// No observation metadata was recorded by the historical writer. + Unknown, + Active, + Absent, + RetainedUnavailable, + Unavailable, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct SourceMetadata { + pub origin: String, + pub status: SourceStatus, + #[serde(default)] + pub pending_update: bool, + pub estimated_tokens: Option, + pub file_paths: Vec, + pub omitted_file_paths: usize, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SourcesObserved { + pub epoch: u32, + pub sources: BTreeMap, +} + +/// Observe only at the ordinary request boundary. Inspection uses the committed +/// result and never invokes a source to refresh these records. +pub fn capture( + observed: &Observation, + snapshot: &BTreeMap, + previous: &BTreeMap, +) -> BTreeMap { + let mut records = previous.clone(); + records.retain(|id, _| snapshot.contains_key(id) || observed.sources.contains_key(id)); + records.extend(observed.sources.iter().map(|(id, value)| { + let status = match value { + Observed::Value(_) => SourceStatus::Active, + Observed::Absent => SourceStatus::Absent, + Observed::Unavailable(_) if snapshot.contains_key(id) => { + SourceStatus::RetainedUnavailable + } + Observed::Unavailable(_) => SourceStatus::Unavailable, + }; + let (file_paths, omitted_file_paths) = if status == SourceStatus::RetainedUnavailable { + previous.get(id).map_or_else( + || (Vec::new(), 0), + |record| (record.file_paths.clone(), record.omitted_file_paths), + ) + } else { + bounded_paths(observed.files.get(id).map_or(&[], Vec::as_slice)) + }; + ( + id.clone(), + SourceMetadata { + origin: if status == SourceStatus::RetainedUnavailable { + previous.get(id).map(|record| record.origin.clone()) + } else { + observed.origins.get(id).cloned() + } + .unwrap_or_else(|| "unknown".into()), + status, + pending_update: observed.pending.contains(id), + estimated_tokens: snapshot.get(id).map(|text| (text.len() / 4) as u64), + file_paths, + omitted_file_paths, + }, + ) + })); + records +} + +fn bounded_paths(paths: &[String]) -> (Vec, usize) { + let mut bytes = 0usize; + let kept: Vec<_> = paths + .iter() + .take(128) + .take_while(|path| { + bytes = bytes.saturating_add(path.len()); + bytes <= 16_384 + }) + .cloned() + .collect(); + let omitted = paths.len() - kept.len(); + (kept, omitted) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provenance_bounds_report_every_omitted_path() { + let paths = vec!["x".repeat(1024); 129]; + let (kept, omitted) = bounded_paths(&paths); + assert_eq!(kept.len(), 16); + assert_eq!(kept.iter().map(String::len).sum::(), 16_384); + assert_eq!(kept.len() + omitted, paths.len()); + } +} diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 40e454cea..6690f261a 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -843,6 +843,15 @@ impl Inner { let payload = EpochStarted { epoch: state.epoch.as_ref().map_or(1, |e| e.number + 1), baseline: context::render_baseline(&snapshot), + sources: super::context_sources::capture( + &observed, + &snapshot, + &state + .epoch + .as_ref() + .map(|epoch| epoch.sources.clone()) + .unwrap_or_default(), + ), snapshot, provider: resolved.provider.clone(), system_prefix: Some(system_prefix), @@ -875,6 +884,9 @@ impl Inner { drop(state); return self.start_epoch(handle, resolved).await; } + observed + .pending + .insert(super::output_style::CONTEXT_KEY.into()); let kept = epoch .snapshot .get(super::output_style::CONTEXT_KEY) @@ -897,13 +909,30 @@ impl Inner { text.push_str("\n\nThe default agent instructions now apply:\n"); text.push_str(&context::base_prompt(resolved.family)); } + let sources = super::context_sources::capture(&observed, &snapshot, &epoch.sources); + let source_update = (sources != epoch.sources).then(|| { + event( + super::context_sources::OBSERVED, + &super::context_sources::SourcesObserved { + epoch: epoch.number, + sources, + }, + ) + }); + let mut updates = Vec::new(); if let Some(text) = text { let payload = ContextUpdated { message_id: cyber_core::ids::new_id("msg"), text, snapshot, }; - self.commit_locked(&mut state, vec![event(CONTEXT_UPDATED, &payload)])?; + updates.push(event(CONTEXT_UPDATED, &payload)); + } + if let Some(source_update) = source_update { + updates.push(source_update); + } + if !updates.is_empty() { + self.commit_locked(&mut state, updates)?; } drop(state); self.instructions_loaded(handle, loaded, "changed").await; @@ -915,6 +944,8 @@ impl Inner { let mut observed = context::observe(&self.context_inputs(&state.info.directory)); let turn = turn_context_for(state, false); for (key, value) in self.options.tools.context_observations_owned(&turn).await { + observed.files.remove(&key); + observed.origins.insert(key.clone(), "tool_host".into()); observed.sources.insert(key, value); } // `goals` → Goal context source. @@ -931,6 +962,13 @@ impl Inner { observed .sources .insert(super::output_style::LANGUAGE_KEY.into(), language); + for key in [ + super::goals::CONTEXT_KEY, + super::output_style::CONTEXT_KEY, + super::output_style::LANGUAGE_KEY, + ] { + observed.origins.insert(key.into(), "runtime".into()); + } observed } diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index 2e06e49cd..d6748e943 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -82,6 +82,7 @@ const ALL: &[&str] = &[ SKILL_FORWARDED, EPOCH_STARTED, CONTEXT_UPDATED, + super::context_sources::OBSERVED, super::deferred_tools::LOADED, SYSTEM_ADDED, SESSION_STARTED, @@ -227,6 +228,8 @@ pub struct EpochStarted { pub epoch: u32, pub baseline: String, pub snapshot: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub sources: BTreeMap, pub provider: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub system_prefix: Option, diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 0dc349add..10c6078a7 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -38,6 +38,8 @@ pub mod computer; mod context; mod context_inspection; mod context_meter; +mod context_sources; +pub use context_sources::{SourceMetadata, SourceStatus}; mod continuation; mod delegations; mod deletion; diff --git a/crates/cyber-server/src/runtime/model.rs b/crates/cyber-server/src/runtime/model.rs index 2e866c5d1..8aef1dd82 100644 --- a/crates/cyber-server/src/runtime/model.rs +++ b/crates/cyber-server/src/runtime/model.rs @@ -338,6 +338,8 @@ pub struct Epoch { pub number: u32, pub baseline: String, pub snapshot: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub sources: BTreeMap, pub provider: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub system_prefix: Option, @@ -1031,6 +1033,14 @@ impl SessionState { } "session.context.epoch_started" => self.on_epoch(decode(e)?), "session.context.updated" => self.on_context_updated(decode(e)?), + "session.context.observed" => { + let observed: super::context_sources::SourcesObserved = decode(e)?; + if let Some(epoch) = &mut self.epoch + && epoch.number == observed.epoch + { + epoch.sources = observed.sources; + } + } "session.system.added" => self.on_system(decode(e)?), "session.structured_output.settled" => { let outcome: super::StructuredOutcome = decode(e)?; @@ -1436,6 +1446,7 @@ impl SessionState { number: e.epoch, baseline: e.baseline, snapshot: e.snapshot, + sources: e.sources, provider: e.provider, system_prefix: e.system_prefix, family: e.family, diff --git a/crates/cyber-server/tests/context_compaction.rs b/crates/cyber-server/tests/context_compaction.rs index d0e838fde..3c88775d7 100644 --- a/crates/cyber-server/tests/context_compaction.rs +++ b/crates/cyber-server/tests/context_compaction.rs @@ -20,6 +20,116 @@ fn small_tail() -> CompactionConfig { } } +#[tokio::test] +async fn inspection_preserves_source_provenance_when_a_later_observation_is_unavailable() { + let h = Harness::new(Setup { + scripts: vec![("test/main", vec![text("first"), text("second")])], + ..Setup::default() + }); + std::fs::write( + h.repo.join("CYBER.md"), + "Private retained instruction body.", + ) + .unwrap(); + let id = h.session().await; + h.runtime.admit(&id, admit("begin")).await.unwrap(); + h.settle(&id).await; + let before = h.runtime.state(&id).await.unwrap(); + let first = serde_json::to_value(h.runtime.inspect_context(&before)).unwrap(); + let source = |inspection: &serde_json::Value| { + inspection["effective_sources"] + .as_array() + .unwrap() + .iter() + .find(|source| source["id"] == "core/instructions") + .unwrap() + .clone() + }; + let active = source(&first); + assert_eq!(active["status"], "active"); + assert_eq!(active["origin"], "instruction_files"); + assert!( + active["file_paths"] + .as_array() + .unwrap() + .iter() + .any(|path| { path.as_str().unwrap().ends_with("CYBER.md") }) + ); + *h.tools.instruction_limits.lock().unwrap() = Err("private diagnostic sentinel".into()); + h.runtime.admit(&id, admit("continue")).await.unwrap(); + h.settle(&id).await; + let after = h.runtime.state(&id).await.unwrap(); + let system_entries = |state: &cyber_server::runtime::SessionState| { + state + .entries + .iter() + .filter(|entry| matches!(entry, cyber_server::runtime::Entry::System { .. })) + .count() + }; + assert_eq!(system_entries(&before), system_entries(&after)); + let inspection = serde_json::to_value(h.runtime.inspect_context(&after)).unwrap(); + let retained = source(&inspection); + assert_eq!(retained["status"], "retained_unavailable"); + assert_eq!(retained["file_paths"], active["file_paths"]); + assert_eq!(retained["estimated_tokens"], active["estimated_tokens"]); + assert_eq!( + before.epoch.unwrap().baseline, + after.epoch.unwrap().baseline + ); + assert!( + !inspection + .to_string() + .contains("private diagnostic sentinel") + ); + assert!( + !inspection + .to_string() + .contains("Private retained instruction body.") + ); + let restarted = h.restart(); + let loaded = restarted.state(&id).await.unwrap(); + assert_eq!( + serde_json::to_value(restarted.inspect_context(&loaded)).unwrap(), + inspection + ); + assert_eq!(h.models.requests("test/main").len(), 2); +} + +#[tokio::test] +async fn historical_source_provenance_stays_unknown_during_read_only_inspection() { + let h = Harness::new(Setup::default()); + let id = h.session().await; + let seq = h.store.aggregate_seq(&id).unwrap().unwrap(); + h.store + .append( + &id, + cyber_store::Expected::Seq(seq), + vec![cyber_store::NewEvent::new( + "session.context.epoch_started.1", + serde_json::json!({ + "epoch": 1, "baseline": "Private old bytes.", + "snapshot": {"legacy/source": "Private old bytes."}, "provider": "test" + }), + )], + ) + .unwrap(); + let restarted = h.restart(); + let state = restarted.state(&id).await.unwrap(); + let inspection = serde_json::to_value(restarted.inspect_context(&state)).unwrap(); + let source = inspection["effective_sources"] + .as_array() + .unwrap() + .iter() + .find(|source| source["id"] == "legacy/source") + .unwrap(); + assert_eq!(source["status"], "unknown"); + assert_eq!(source["origin"], "unknown"); + assert!(source["estimated_tokens"].as_u64().unwrap() > 0); + assert!(!inspection.to_string().contains("Private old bytes.")); + assert!(h.models.requests("test/main").is_empty()); + assert_eq!(h.store.aggregate_seq(&id).unwrap().unwrap(), seq + 1); +} + #[tokio::test] async fn historical_epoch_without_a_system_prefix_keeps_legacy_request_bytes() { let h = Harness::new(Setup { diff --git a/crates/cyber-server/tests/http.rs b/crates/cyber-server/tests/http.rs index 058944eca..c924c4b5d 100644 --- a/crates/cyber-server/tests/http.rs +++ b/crates/cyber-server/tests/http.rs @@ -4083,6 +4083,13 @@ async fn context_inspection_is_authenticated_read_only_and_metadata_only_by_defa assert!(data["baseline"].is_null()); assert!(data["summary"].is_null()); let estimate = &data["inspection"]; + let effective = estimate["effective_sources"].as_array().unwrap(); + let date = effective + .iter() + .find(|source| source["id"] == "core/date") + .unwrap(); + assert_eq!(date["status"], "active"); + assert_eq!(date["origin"], "runtime"); assert_eq!(estimate["epoch_pending"], false); assert_eq!(estimate["last_request_tokens"], 100); assert_eq!(estimate["window"], 200_000); @@ -4098,8 +4105,9 @@ async fn context_inspection_is_authenticated_read_only_and_metadata_only_by_defa pair[0]["estimated_tokens"].as_u64().unwrap() >= pair[1]["estimated_tokens"].as_u64().unwrap() })); - let explicit: Value = inspect() - .query(&[("include_content", true)]) + let explicit: Value = client + .get(format!("{url}?include_content=true")) + .basic_auth("cyber", Some("pw")) .send() .await .unwrap() diff --git a/crates/cyber-tui/src/context_inspection.rs b/crates/cyber-tui/src/context_inspection.rs index 2864e6855..43dcd2ae4 100644 --- a/crates/cyber-tui/src/context_inspection.rs +++ b/crates/cyber-tui/src/context_inspection.rs @@ -56,5 +56,70 @@ pub(crate) fn format(data: &Value) -> String { } } } + lines.extend(effective_sources(inspection)); lines.join("\n") } + +fn effective_sources(inspection: &Value) -> Vec { + let mut lines = vec![String::new(), "Recorded context sources".into()]; + for source in inspection["effective_sources"] + .as_array() + .into_iter() + .flatten() + { + let field = |name| crate::messaging::clean_line(source[name].as_str().unwrap_or("unknown")); + let tokens = source["estimated_tokens"] + .as_u64() + .map_or_else(|| "unknown".into(), |n| format!("~{n}")); + lines.push(format!( + "{} · {} · {} · {tokens} tokens", + field("id"), + field("origin"), + field("status").replace('_', " ") + )); + if source["pending_update"].as_bool() == Some(true) { + lines.push(" Update pending at the next safe boundary".into()); + } + for path in source["file_paths"].as_array().into_iter().flatten() { + lines.push(format!( + " {}", + crate::messaging::clean_line(path.as_str().unwrap_or("")) + )); + } + let omitted = source["omitted_file_paths"].as_u64().unwrap_or(0); + if omitted > 0 { + lines.push(format!(" Additional paths omitted · {omitted}")); + } + } + let omitted = inspection["omitted_sources"].as_u64().unwrap_or(0); + if omitted > 0 { + lines.push(format!("Additional sources omitted · {omitted}")); + } + lines +} + +#[cfg(test)] +mod tests { + #[test] + fn inspection_displays_pending_and_retained_sources_with_safe_path_text() { + let text = super::format(&serde_json::json!({ + "epoch": 3, + "inspection": { + "effective_sources": [{ + "id": "core/instructions", "origin": "instruction_files", + "status": "retained_unavailable", "estimated_tokens": 42, + "pending_update": true, "file_paths": ["/repo/CYBER\n.md\u{001b}"], + "omitted_file_paths": 2 + }], + "omitted_sources": 1 + } + })); + assert!(text.contains("Estimated window use: unknown")); + assert!(text.contains("retained unavailable · ~42 tokens")); + assert!(text.contains("Update pending at the next safe boundary")); + assert!(text.contains("/repo/CYBER .md")); + assert!(text.contains("Additional paths omitted · 2")); + assert!(text.contains("Additional sources omitted · 1")); + assert!(!text.contains('\u{001b}')); + } +} diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index 3fd3b6115..a17d8f5d9 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -43,9 +43,33 @@ check, SDK typecheck and 136 SDK tests (ten configured skips). Package library Clippy passed after routing inspection through the existing loader. Strict OpenSpec validation passed 100 items and lint reported zero errors/25 warnings. Native macOS C deprecation and linker unwind-size warnings were observed. -Source-bound native CI remains pending on preceding revision `a4d28df4`. +The native run for preceding revision `a4d28df4` completed as cancelled. This continuation has no native/platform acceptance. +### Recorded source provenance continuation + +Epoch events now capture source origin, observation status and bounded file +provenance from the actual observation. A separate Epoch-bound observation event +records metadata changes without adding transcript messages. Unavailable sources +retain the admitted value and its provenance. Historical sources without metadata +report unknown origin and status. Tool-host overrides record their own origin and +drop file provenance from the replaced built-in source. Deferred output-style +changes carry a pending flag. + +Inspection lists up to 256 sources, largest first, with explicit omitted counts. +Each file inventory is bounded to 128 paths and 16 KiB. Content and unavailable +diagnostic text stay outside the metadata response. The next-request estimate, +config/revocation provenance and complete pending-change acceptance remain open. + +Local verification passed all 24 context/compaction integration cases, the +standalone authenticated HTTP inspection case, the source-path bounds unit case, +OpenAPI regeneration and SDK consistency/typecheck with 136 passing tests and ten +configured skips. Server/TUI all-target Clippy and both focused TUI rendering/stale-reply tests passed. Strict OpenSpec passed all 100 +items; cross-spec lint reported zero errors and 25 warnings. Native workflow YAML +parsed successfully. The focused review corrected source attribution when a tool +host replaces a built-in source, retained unavailable provenance, and source +sorting before response truncation. Platform acceptance remains open. + ### Zero-byte ordinary restore continuation Ordinary cleanup now records payload completion for exact empty inventories and diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 29d85c56e..70ef66a85 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1227,7 +1227,7 @@ The runtime resolves settings after the read and verifies its captured Location, ### Inspecting the current context -Open `/context` in the TUI to see the stored Context Epoch, projected source groups, estimated token counts and the last measured request. The estimate uses the persisted system text and projected conversation. Tool schemas and instructions added only during a turn are excluded. The panel also reports whether an Epoch rebuild is pending, how many inputs are queued or held, and how many scoped rules were delivered in the Epoch. +Open `/context` in the TUI to see the stored Context Epoch, projected source groups, estimated token counts and the last measured request. The estimate uses the persisted system text and projected conversation. Tool schemas and instructions added only during a turn are excluded. The panel also reports whether an Epoch rebuild is pending, how many inputs are queued or held, and how many scoped rules were delivered in the Epoch. Individual sources show their last recorded origin, status, estimate and instruction file paths. An unavailable observation retains the admitted source and its provenance. Historical sources without recorded provenance show unknown status and origin. Output-style changes deferred during a turn are marked pending. Inspection reports up to 256 sources, largest first, and up to 128 file paths per source within 16 KiB, with explicit omission counts. Authenticated clients can use `GET /api/v1/sessions/{sessionID}/context` for the same metadata. Pass `include_content=true` to retrieve the stored baseline, compaction summary and task text. Inspection reads stored Session state without starting a turn or reloading instruction files. diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index 47c1905af..f44df4e12 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -130,7 +130,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - [ ] 4.1 Implement trusted path-scoped rule discovery/precedence, validated globs, overrides and bounded instruction assembly without changing `instructions` array compatibility. - [ ] 4.2 Bind reminder delivery to successful authorized operations and Context Epoch identity, including edits/patches, partial settlements, compaction/restart and changed/revoked rules. - [ ] 4.3 Add authenticated read-only context inspection and `/context`, source estimates/origins/status/pending updates and honest utilization; synchronize OpenAPI/SDK and doctor diagnostics. - - The Session endpoint and TUI panel expose stored source groups and an estimate based on request replay. The default response omits stored text. Individual source provenance, pending source changes and next-request estimates remain open. + - The Session endpoint and TUI panel expose stored source groups and an estimate based on request replay. The default response omits stored text. Individual source metadata now persists with Epoch creation and ordinary observation, including retained-unavailable state, bounded file provenance, historical unknowns and deferred output-style updates. Remaining pending source changes and complete next-request estimates remain open. - [ ] 4.4 Test no rule leakage across Location/Epoch, denied/failed calls, truncation, untrusted sources, stale response and token-count consistency with actual request assembly. ## 5. Root structured output diff --git a/sdk/openapi.json b/sdk/openapi.json index 188a229a4..47553deb1 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -9758,6 +9758,18 @@ "ContextInspection": { "type": "object", "properties": { + "effective_sources": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EffectiveContextSource" + }, + "description": "Latest recorded source observations, without source content." + }, + "omitted_sources": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, "sources": { "type": "array", "items": { @@ -9819,6 +9831,8 @@ } }, "required": [ + "effective_sources", + "omitted_sources", "sources", "epoch_pending", "queued_inputs", @@ -9826,6 +9840,68 @@ "delivered_rule_count" ] }, + "EffectiveContextSource": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "origin": { + "type": "string" + }, + "status": { + "$ref": "#/components/schemas/SourceStatus" + }, + "pending_update": { + "type": "boolean", + "default": false + }, + "estimated_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "file_paths": { + "type": "array", + "items": { + "type": "string" + } + }, + "omitted_file_paths": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "id", + "origin", + "status", + "file_paths", + "omitted_file_paths" + ] + }, + "SourceStatus": { + "oneOf": [ + { + "type": "string", + "enum": [ + "active", + "absent", + "retained_unavailable", + "unavailable" + ] + }, + { + "type": "string", + "const": "unknown", + "description": "No observation metadata was recorded by the historical writer." + } + ] + }, "ContextSourceEstimate": { "type": "object", "properties": { diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 230cbf600..4dd9082ea 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -737,6 +737,9 @@ export type Content2 = { }; export type ContextInspection = { + /** Latest recorded source observations, without source content. */ + effective_sources: EffectiveContextSource[]; + omitted_sources: number; /** Sources in the stored projection, ordered by estimated size. */ sources: ContextSourceEstimate[]; /** @@ -1272,6 +1275,16 @@ export type EditGoal = { /** The tool shape the model edits files with. */ export type EditTool = "freeform_patch" | "json_patch" | "edit"; +export type EffectiveContextSource = { + id: string; + origin: string; + status: SourceStatus; + pending_update?: boolean; + estimated_tokens?: number | null; + file_paths: string[]; + omitted_file_paths: number; +}; + /** What happens to a derived effort the model does not list. */ export type EffortClamp = "nearest_lower" | "fail"; @@ -5016,6 +5029,8 @@ export type Source = { export type Source2 = "live" | "draft"; +export type SourceStatus = "active" | "absent" | "retained_unavailable" | "unavailable" | "unknown"; + /** A user statement with its source, kept outside generated prose. */ export type Sourced = { message_id: string; From 64052cbac7d1793b6eaf9ed5bee30490a9fbdf05 Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 23:25:36 -0400 Subject: [PATCH 85/87] Record context estimates from assembled provider requests --- .../src/runtime/context_inspection.rs | 74 +++++++++++- crates/cyber-server/src/runtime/drain.rs | 7 ++ crates/cyber-server/src/runtime/events.rs | 2 + crates/cyber-server/src/runtime/mod.rs | 2 +- crates/cyber-server/src/runtime/model.rs | 4 + crates/cyber-server/tests/http.rs | 7 ++ crates/cyber-server/tests/structured_root.rs | 58 ++++++++++ crates/cyber-tui/src/context_inspection.rs | 53 +++++++++ docs/goal-delivery.md | 28 ++++- docs/local-core-guide.md | 2 + .../complete-daily-coding-surfaces/tasks.md | 2 +- sdk/openapi.json | 107 ++++++++++++++---- sdk/typescript/src/generated/types.ts | 17 +++ 13 files changed, 334 insertions(+), 29 deletions(-) diff --git a/crates/cyber-server/src/runtime/context_inspection.rs b/crates/cyber-server/src/runtime/context_inspection.rs index 1a8b346e5..d77e27e68 100644 --- a/crates/cyber-server/src/runtime/context_inspection.rs +++ b/crates/cyber-server/src/runtime/context_inspection.rs @@ -2,7 +2,7 @@ use cyber_llm::Message; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use super::context_sources::{SourceMetadata, SourceStatus}; use super::{InputStatus, Runtime, SessionState, view}; @@ -14,15 +14,76 @@ pub struct EffectiveContextSource { pub metadata: SourceMetadata, } -#[derive(Debug, Serialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ContextSourceEstimate { pub id: String, pub origin: String, pub estimated_tokens: u64, } +/// The complete input estimate captured from a final assembled provider request. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PreparedContextEstimate { + pub step_id: String, + pub epoch: u32, + pub provider: String, + pub model: String, + pub window: Option, + pub estimated_input_tokens: u64, + pub estimated_input_utilization: Option, + pub sources: Vec, +} + +impl PreparedContextEstimate { + pub(super) fn capture( + request: &cyber_llm::LlmRequest, + resolved: &super::ResolvedModel, + epoch: u32, + compacted: bool, + step_id: String, + ) -> Self { + let window = (resolved.context_limit > 0).then_some(resolved.context_limit); + let estimated_input_tokens = + view::estimate_tokens(&request.system, &request.messages, &request.tools); + let mut sources = estimates(&request.system[..2], &request.messages, compacted); + let turn_bytes = request + .system + .iter() + .skip(2) + .map(String::len) + .sum::(); + if turn_bytes > 0 { + sources.push(source( + "turn_instructions", + "request_assembly", + (turn_bytes / 4) as u64, + )); + } + sources.push(source( + "tool_schemas", + "request_assembly", + serde_json::to_string(&request.tools).map_or(0, |text| (text.len() / 4) as u64), + )); + sort_sources(&mut sources); + Self { + step_id, + epoch, + provider: resolved.provider.clone(), + model: resolved.model.clone(), + window, + estimated_input_tokens, + estimated_input_utilization: window + .map(|window| estimated_input_tokens as f64 / window as f64), + sources, + } + } +} + #[derive(Debug, Serialize, JsonSchema)] pub struct ContextInspection { + /// Final assembly of the most recently prepared request, including schemas + /// and turn instructions. Historical events may have no estimate. + pub last_prepared_request: Option, /// Latest recorded source observations, without source content. pub effective_sources: Vec, pub omitted_sources: usize, @@ -49,6 +110,7 @@ impl Runtime { .as_ref() .and_then(|model| (model.context_limit > 0).then_some(model.context_limit)); let mut inspection = ContextInspection { + last_prepared_request: state.prepared_context.clone(), effective_sources: Vec::new(), omitted_sources: 0, sources: Vec::new(), @@ -131,7 +193,7 @@ fn count_inputs(state: &SessionState, status: InputStatus) -> usize { } fn estimates( - system: &[String; 2], + system: &[String], messages: &[Message], compacted: bool, ) -> Vec { @@ -160,12 +222,16 @@ fn estimates( view::estimate_tokens(&[], history, &[]), )); } + sort_sources(&mut sources); + sources +} + +fn sort_sources(sources: &mut [ContextSourceEstimate]) { sources.sort_by(|a, b| { b.estimated_tokens .cmp(&a.estimated_tokens) .then(a.id.cmp(&b.id)) }); - sources } fn source(id: &str, origin: &str, estimated_tokens: u64) -> ContextSourceEstimate { diff --git a/crates/cyber-server/src/runtime/drain.rs b/crates/cyber-server/src/runtime/drain.rs index 6690f261a..dd74164d6 100644 --- a/crates/cyber-server/src/runtime/drain.rs +++ b/crates/cyber-server/src/runtime/drain.rs @@ -1291,6 +1291,13 @@ impl Inner { let message_id = cyber_core::ids::new_id("msg"); let origin = super::usage_origin::step_origin(&self.store, &state)?; let payload = StepStarted { + context_estimate: Some(super::context_inspection::PreparedContextEstimate::capture( + &request, + resolved, + epoch.number, + state.compacted.is_some(), + message_id.clone(), + )), input_message_ids: super::requests::input_owners::capture(&state), agent: Some(state.info.agent.clone()), mode: Some(mode), diff --git a/crates/cyber-server/src/runtime/events.rs b/crates/cyber-server/src/runtime/events.rs index d6748e943..90b1db599 100644 --- a/crates/cyber-server/src/runtime/events.rs +++ b/crates/cyber-server/src/runtime/events.rs @@ -268,6 +268,8 @@ pub struct SystemAdded { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StepStarted { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context_estimate: Option, /// Complete active input cohort captured before this inference step. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub input_message_ids: Vec, diff --git a/crates/cyber-server/src/runtime/mod.rs b/crates/cyber-server/src/runtime/mod.rs index 10c6078a7..b0e2696d9 100644 --- a/crates/cyber-server/src/runtime/mod.rs +++ b/crates/cyber-server/src/runtime/mod.rs @@ -248,7 +248,7 @@ pub use auto::{AutoDecision, AutoEffect, AutoReview, AutoReviewStage, TAINTED_RE pub use bus::LiveEvent; pub use compaction::CompactionConfig; pub use context::{ContextInputs, Observed as ContextObservation, base_prompt}; -pub use context_inspection::ContextInspection; +pub use context_inspection::{ContextInspection, PreparedContextEstimate}; pub use context_meter::ContextMeter; pub use cyber_core::prompts::ModelFamily; pub use events::{AuxiliaryUsage, CompactionTrigger, registry as event_registry}; diff --git a/crates/cyber-server/src/runtime/model.rs b/crates/cyber-server/src/runtime/model.rs index 8aef1dd82..002667548 100644 --- a/crates/cyber-server/src/runtime/model.rs +++ b/crates/cyber-server/src/runtime/model.rs @@ -556,6 +556,8 @@ pub struct SessionState { /// Boxed: Session state is held in deeply nested futures, so it stays small. #[serde(skip)] pub last_step: Option>, + #[serde(skip)] + pub(crate) prepared_context: Option, pub steps: Vec, /// File diffs by user message. pub diffs: BTreeMap>, @@ -823,6 +825,7 @@ impl SessionState { turn_mode: None, turn_agent: None, last_step: None, + prepared_context: None, steps: Vec::new(), diffs: BTreeMap::new(), revert: None, @@ -1480,6 +1483,7 @@ impl SessionState { fn on_step_started(&mut self, s: StepStarted, time_ms: i64) -> Result<(), String> { super::requests::input_owners::validate(self, &s.input_message_ids)?; + self.prepared_context = s.context_estimate; self.first_turn_started = true; self.turn_mode = Some(s.mode.unwrap_or_else(|| self.info.mode.clone())); let agent = s.agent.unwrap_or_else(|| self.info.agent.clone()); diff --git a/crates/cyber-server/tests/http.rs b/crates/cyber-server/tests/http.rs index c924c4b5d..1fbd69f90 100644 --- a/crates/cyber-server/tests/http.rs +++ b/crates/cyber-server/tests/http.rs @@ -4073,6 +4073,7 @@ async fn context_inspection_is_authenticated_read_only_and_metadata_only_by_defa fresh["data"]["inspection"]["estimated_input_tokens"], Value::Null ); + assert!(fresh["data"]["inspection"]["last_prepared_request"].is_null()); assert!(h.models.requests("test/main").is_empty()); api.post(&format!("/sessions/{id}/prompt"), prompt("hi")) @@ -4092,6 +4093,12 @@ async fn context_inspection_is_authenticated_read_only_and_metadata_only_by_defa assert_eq!(date["origin"], "runtime"); assert_eq!(estimate["epoch_pending"], false); assert_eq!(estimate["last_request_tokens"], 100); + assert!( + estimate["last_prepared_request"]["estimated_input_tokens"] + .as_u64() + .unwrap() + > 0 + ); assert_eq!(estimate["window"], 200_000); let tokens = estimate["estimated_input_tokens"].as_u64().unwrap(); assert!(tokens > 0); diff --git a/crates/cyber-server/tests/structured_root.rs b/crates/cyber-server/tests/structured_root.rs index a393de327..5bafef734 100644 --- a/crates/cyber-server/tests/structured_root.rs +++ b/crates/cyber-server/tests/structured_root.rs @@ -52,6 +52,64 @@ fn harness(main: Vec>) -> Harness { }) } +#[tokio::test] +async fn inspected_request_estimate_includes_the_finalizer_and_every_assembled_input() { + let h = harness(vec![structured("final", r#"{"summary":"done"}"#)]); + let id = h.session().await; + let mut output_schema = schema(); + output_schema["description"] = "Describe the completed audit. ".repeat(200).into(); + ask(&h, &id, "audit this workspace", Some(output_schema)).await; + h.settle(&id).await; + let requests = h.models.requests("test/main"); + assert_eq!(requests.len(), 1); + let request = &requests[0]; + assert!(request.system.len() > 2); + assert!( + request + .tools + .iter() + .any(|tool| tool.name == "structured_output") + ); + let serialized_bytes = request.system.iter().map(String::len).sum::() + + serde_json::to_vec(&request.messages).unwrap().len() + + serde_json::to_vec(&request.tools).unwrap().len(); + let state = h.state(&id).await; + let estimate = h + .runtime + .inspect_context(&state) + .last_prepared_request + .unwrap(); + assert_eq!( + estimate.estimated_input_tokens, + (serialized_bytes / 4) as u64 + ); + assert_eq!(estimate.step_id, state.steps.last().unwrap().step_id); + assert_eq!( + estimate.estimated_input_utilization, + Some(estimate.estimated_input_tokens as f64 / 200_000.0) + ); + assert!( + estimate + .sources + .iter() + .any(|source| source.id == "turn_instructions" && source.estimated_tokens > 0) + ); + assert!( + estimate + .sources + .iter() + .any(|source| source.id == "tool_schemas" && source.estimated_tokens > 1000) + ); + let restarted = h.restart(); + assert_eq!( + restarted + .inspect_context(&restarted.state(&id).await.unwrap()) + .last_prepared_request, + Some(estimate) + ); + assert_eq!(h.models.requests("test/main").len(), 1); +} + #[tokio::test] async fn lost_terminal_commit_ack_reopens_the_original_outcome_without_dispatch() { use std::sync::atomic::Ordering; diff --git a/crates/cyber-tui/src/context_inspection.rs b/crates/cyber-tui/src/context_inspection.rs index 43dcd2ae4..5e654fe81 100644 --- a/crates/cyber-tui/src/context_inspection.rs +++ b/crates/cyber-tui/src/context_inspection.rs @@ -56,10 +56,53 @@ pub(crate) fn format(data: &Value) -> String { } } } + lines.extend(prepared_request(&inspection["last_prepared_request"])); lines.extend(effective_sources(inspection)); lines.join("\n") } +fn prepared_request(prepared: &Value) -> Vec { + if !prepared.is_object() { + return Vec::new(); + } + let mut lines = vec![ + String::new(), + format!( + "Last prepared request · {}", + crate::messaging::clean_line(prepared["step_id"].as_str().unwrap_or("unknown")) + ), + format!( + "Estimated complete input · {} tokens", + prepared["estimated_input_tokens"] + .as_u64() + .map_or_else(|| "unknown".into(), |n| n.to_string()) + ), + format!( + "Model · {}/{} · Epoch {}", + crate::messaging::clean_line(prepared["provider"].as_str().unwrap_or("unknown")), + crate::messaging::clean_line(prepared["model"].as_str().unwrap_or("unknown")), + prepared["epoch"] + .as_u64() + .map_or_else(|| "unknown".into(), |n| n.to_string()) + ), + format!( + "Estimated prepared window use · {}", + prepared["estimated_input_utilization"] + .as_f64() + .map_or_else( + || "unknown".into(), + |fraction| format!("{:.1}%", fraction * 100.0) + ) + ), + ]; + for source in prepared["sources"].as_array().into_iter().flatten() { + let id = crate::messaging::clean_line(source["id"].as_str().unwrap_or("unknown")); + let tokens = source["estimated_tokens"].as_u64().unwrap_or(0); + lines.push(format!("{id} · ~{tokens} tokens")); + } + lines +} + fn effective_sources(inspection: &Value) -> Vec { let mut lines = vec![String::new(), "Recorded context sources".into()]; for source in inspection["effective_sources"] @@ -105,6 +148,11 @@ mod tests { let text = super::format(&serde_json::json!({ "epoch": 3, "inspection": { + "last_prepared_request": { + "step_id": "msg_1", "epoch": 2, "provider": "test", "model": "main", + "estimated_input_tokens": 1280, "estimated_input_utilization": 0.5, + "sources": [{"id": "tool_schemas", "estimated_tokens": 280}] + }, "effective_sources": [{ "id": "core/instructions", "origin": "instruction_files", "status": "retained_unavailable", "estimated_tokens": 42, @@ -115,6 +163,11 @@ mod tests { } })); assert!(text.contains("Estimated window use: unknown")); + assert!(text.contains("Last prepared request · msg_1")); + assert!(text.contains("Estimated complete input · 1280 tokens")); + assert!(text.contains("Model · test/main · Epoch 2")); + assert!(text.contains("Estimated prepared window use · 50.0%")); + assert!(text.contains("tool_schemas · ~280 tokens")); assert!(text.contains("retained unavailable · ~42 tokens")); assert!(text.contains("Update pending at the next safe boundary")); assert!(text.contains("/repo/CYBER .md")); diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index a17d8f5d9..c0a50bd81 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -21,8 +21,8 @@ section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | | --- | --- | | #93 rewind/snapshots, tasks 2.4/3.2–3.4 and M2–M8 | Local worker origins, reviewed keep/rollback, paired Store settlement, native acknowledgements, CLI/TUI recovery and several payload-reclamation paths exist. Managed Git partial-kill and fully prepared cancellation cleanup acceptance pass locally, including typed private links, but default Git admission remains gated. Finish partial resource construction, complete reference/CAS/metadata and managed/Git reclamation, remaining crash cuts, platform/client acceptance and delivery. Preserve legacy unknown fences. | -| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads, edits and patches, including intercepted shell patches, use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native operation and dispatched attempt. A partial patch can deliver rules for completed files. Finish exact config-generation and revocation/restart provenance. The authenticated Session context route and TUI `/context` now show stored source groups, projected token estimates, measured usage and pending Epoch state. The default response hides stored text. Finish per-source provenance, revocation and pending source changes, and complete leakage, restart and native acceptance. | -| #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI and 16 structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | +| #93 rules/context, tasks 4.1–4.4 | Local `instruction_limits` validation and runtime/doctor wiring now bound complete rendered baselines/reminders, including labels/notices and bounded file lists with exact delivery identities. Local fenced/trusted rule discovery, validated relative globs, baseline/scoped-read separation and typed delivery bindings now exist. Built-in reads, edits and patches, including intercepted shell patches, use actual native file receipts after hook rewriting; nested delivery reloads settings and checks Location, native root/file binding and Epoch before discovery and settlement. Durable rule delivery records the native operation and dispatched attempt. A partial patch can deliver rules for completed files. Finish exact config-generation and revocation/restart provenance. The authenticated Session context route and TUI `/context` now show stored source groups, projected token estimates, measured usage and pending Epoch state. The default response hides stored text. Recorded source provenance/status, unavailable retention and deferred style updates now survive restart. The last prepared request includes complete assembled input estimates. Finish next-request projection, revocation and remaining pending source changes, and complete leakage and native acceptance. | +| #93 structured results, tasks 5.1–5.4 | Local functional coverage now includes compaction, retry/budget/interruption handling, terminal-writer recovery, physical Store reopen after committed acknowledgement loss, and atomic ownership under steering. Five real CLI cases passed previously and all 17 current structured-root cases pass; current-source platform acceptance remains required before accepting 5.4. | | #93 JSONL, tasks 6.1–6.4 | #117 merged basic typed parsing, correlation, replies and EOF draining. Complete exact driver ownership, bounded retained state and output backpressure, hook-event output, safe atomic last-message publication, and steering/capacity/SIGINT/restart/lost-ack/shared-Session acceptance. Task 6.2 is reopened on inspected code evidence below. | | #92 native enforcement, tasks 2/N/R | Foundation/probe evidence exists, but production launch still uses the legacy sandbox route. Finish generation/image-bound native admission, revocation/quarantine/lifetime, independently intersected capabilities, real tool execution and native runtime/platform positives/denials. Linux R1 direction/support review and later R6 activation remain distinct gates. | | #92 protocol/callbacks/producers, tasks 3–6/S/D | Finish contribution ownership, hooks/providers/catalog transforms and TypeScript kit, durable scoped callbacks (`session/prompt` still explicitly refuses), Setup and DirectoryAdded plus combined producer acceptance. Preserve accepted FileChanged and CwdChanged fixes. Complete S1 epoch/directory-key/legacy-deletion and D1 directory-admission/Safe Boundary/child-policy review before their dependent code. | @@ -70,6 +70,30 @@ parsed successfully. The focused review corrected source attribution when a tool host replaces a built-in source, retained unavailable provenance, and source sorting before response truncation. Platform acceptance remains open. +### Prepared request estimate continuation + +The final request assembly now captures its input estimate in the durable +`StepStarted` event. It includes projected attachments, structured-output +instructions, deferred tool descriptions and callable schemas. Replay retains +only the most recent estimate in Session state. Historical step events retain +an explicit unknown estimate. API/TUI inspection identifies the prepared step, +model and Epoch separately from the current stored projection and measured +provider usage. It performs no new source observations or model calls. + +The scripted-provider regression compares the estimate with the serialized +request that the adapter actually received, including a large structured-output +schema, and verifies restart without dispatch. All 17 structured-root and 24 +context/compaction cases pass locally. Complete next-request projection remains +open because configuration, tool visibility and pending sources can change at +the next ordinary request boundary. + +The authenticated HTTP metadata/content case, the TUI prepared-request rendering +case, OpenAPI regeneration and SDK consistency/typecheck passed. SDK tests passed +136 cases with ten configured skips. Server/TUI all-target Clippy passed with +warnings denied. Strict OpenSpec passed 100 items and lint reported zero errors +with 25 warnings. Native C deprecation and linker unwind-size warnings were +observed during compilation. Current-source native acceptance remains required. + ### Zero-byte ordinary restore continuation Ordinary cleanup now records payload completion for exact empty inventories and diff --git a/docs/local-core-guide.md b/docs/local-core-guide.md index 70ef66a85..d62f81a75 100644 --- a/docs/local-core-guide.md +++ b/docs/local-core-guide.md @@ -1229,6 +1229,8 @@ The runtime resolves settings after the read and verifies its captured Location, Open `/context` in the TUI to see the stored Context Epoch, projected source groups, estimated token counts and the last measured request. The estimate uses the persisted system text and projected conversation. Tool schemas and instructions added only during a turn are excluded. The panel also reports whether an Epoch rebuild is pending, how many inputs are queued or held, and how many scoped rules were delivered in the Epoch. Individual sources show their last recorded origin, status, estimate and instruction file paths. An unavailable observation retains the admitted source and its provenance. Historical sources without recorded provenance show unknown status and origin. Output-style changes deferred during a turn are marked pending. Inspection reports up to 256 sources, largest first, and up to 128 file paths per source within 16 KiB, with explicit omission counts. +The last prepared request has a separate complete estimate captured after attachments, turn instructions, deferred tool descriptions and callable schemas are assembled. Its step ID, model and Epoch identify the request it describes. It can precede a model call that was later interrupted. The stored projection remains an estimate of the current recorded conversation. Provider token usage appears separately when the request finishes. Individual source estimates use independent rounding. + Authenticated clients can use `GET /api/v1/sessions/{sessionID}/context` for the same metadata. Pass `include_content=true` to retrieve the stored baseline, compaction summary and task text. Inspection reads stored Session state without starting a turn or reloading instruction files. ### Switches diff --git a/openspec/changes/complete-daily-coding-surfaces/tasks.md b/openspec/changes/complete-daily-coding-surfaces/tasks.md index f44df4e12..1be27e09f 100644 --- a/openspec/changes/complete-daily-coding-surfaces/tasks.md +++ b/openspec/changes/complete-daily-coding-surfaces/tasks.md @@ -130,7 +130,7 @@ proof/reconciliation for these cuts before claiming complete managed recovery. - [ ] 4.1 Implement trusted path-scoped rule discovery/precedence, validated globs, overrides and bounded instruction assembly without changing `instructions` array compatibility. - [ ] 4.2 Bind reminder delivery to successful authorized operations and Context Epoch identity, including edits/patches, partial settlements, compaction/restart and changed/revoked rules. - [ ] 4.3 Add authenticated read-only context inspection and `/context`, source estimates/origins/status/pending updates and honest utilization; synchronize OpenAPI/SDK and doctor diagnostics. - - The Session endpoint and TUI panel expose stored source groups and an estimate based on request replay. The default response omits stored text. Individual source metadata now persists with Epoch creation and ordinary observation, including retained-unavailable state, bounded file provenance, historical unknowns and deferred output-style updates. Remaining pending source changes and complete next-request estimates remain open. + - The Session endpoint and TUI panel expose stored source groups and an estimate based on request replay. The default response omits stored text. Individual source metadata now persists with Epoch creation and ordinary observation, including retained-unavailable state, bounded file provenance, historical unknowns and deferred output-style updates. The last prepared request now records the complete assembly estimate, including callable schemas and turn instructions, for comparison with actual provider input. Remaining pending source changes and complete next-request estimates remain open. - [ ] 4.4 Test no rule leakage across Location/Epoch, denied/failed calls, truncation, untrusted sources, stale response and token-count consistency with actual request assembly. ## 5. Root structured output diff --git a/sdk/openapi.json b/sdk/openapi.json index 47553deb1..b733527fb 100644 --- a/sdk/openapi.json +++ b/sdk/openapi.json @@ -9758,6 +9758,17 @@ "ContextInspection": { "type": "object", "properties": { + "last_prepared_request": { + "anyOf": [ + { + "$ref": "#/components/schemas/PreparedContextEstimate" + }, + { + "type": "null" + } + ], + "description": "Final assembly of the most recently prepared request, including schemas\nand turn instructions. Historical events may have no estimate." + }, "effective_sources": { "type": "array", "items": { @@ -9840,6 +9851,81 @@ "delivered_rule_count" ] }, + "PreparedContextEstimate": { + "type": "object", + "properties": { + "step_id": { + "type": "string" + }, + "epoch": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "provider": { + "type": "string" + }, + "model": { + "type": "string" + }, + "window": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "estimated_input_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "estimated_input_utilization": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "sources": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ContextSourceEstimate" + } + } + }, + "required": [ + "step_id", + "epoch", + "provider", + "model", + "estimated_input_tokens", + "sources" + ], + "description": "The complete input estimate captured from a final assembled provider request." + }, + "ContextSourceEstimate": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "origin": { + "type": "string" + }, + "estimated_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "id", + "origin", + "estimated_tokens" + ] + }, "EffectiveContextSource": { "type": "object", "properties": { @@ -9902,27 +9988,6 @@ } ] }, - "ContextSourceEstimate": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "origin": { - "type": "string" - }, - "estimated_tokens": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - "required": [ - "id", - "origin", - "estimated_tokens" - ] - }, "Data_Array_of_FileDiff": { "type": "object", "properties": { diff --git a/sdk/typescript/src/generated/types.ts b/sdk/typescript/src/generated/types.ts index 4dd9082ea..b84247717 100644 --- a/sdk/typescript/src/generated/types.ts +++ b/sdk/typescript/src/generated/types.ts @@ -737,6 +737,11 @@ export type Content2 = { }; export type ContextInspection = { + /** + * Final assembly of the most recently prepared request, including schemas + * and turn instructions. Historical events may have no estimate. + */ + last_prepared_request?: PreparedContextEstimate | null; /** Latest recorded source observations, without source content. */ effective_sources: EffectiveContextSource[]; omitted_sources: number; @@ -3760,6 +3765,18 @@ export type PreparationReceiptData = { data: SnapshotPreparationReceipt | null; }; +/** The complete input estimate captured from a final assembled provider request. */ +export type PreparedContextEstimate = { + step_id: string; + epoch: number; + provider: string; + model: string; + window?: number | null; + estimated_input_tokens: number; + estimated_input_utilization?: number | null; + sources: ContextSourceEstimate[]; +}; + /** Product telemetry is not implemented and always off. */ export type ProductTelemetry = { enabled: boolean; From 2a628f12bd1b7a027cd489308de1403fa7d0faca Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 23:32:02 -0400 Subject: [PATCH 86/87] Gate native retirement helpers with their platform consumers --- .github/workflows/binary-editor-proof.yml | 1 + .../activity/snapshot/acknowledgement.rs | 4 ++++ docs/goal-delivery.md | 22 +++++++++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/binary-editor-proof.yml b/.github/workflows/binary-editor-proof.yml index 762e1508b..19d8eb395 100644 --- a/.github/workflows/binary-editor-proof.yml +++ b/.github/workflows/binary-editor-proof.yml @@ -3,6 +3,7 @@ on: pull_request: paths: - crates/cyber-core/src/binary/** + - crates/cyber-core/src/worktrees/** - crates/cyber-core/Cargo.toml - Cargo.lock - .github/workflows/binary-editor-proof.yml diff --git a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs index 0ecaf6417..90ff1ad61 100644 --- a/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs +++ b/crates/cyber-core/src/worktrees/activity/snapshot/acknowledgement.rs @@ -69,15 +69,18 @@ fn stage_owner(name: &str) -> io::Result<&str> { Ok(owner) } impl Acknowledged { + #[cfg(any(target_os = "linux", target_os = "macos"))] pub(super) fn retirement_context(&self) -> (String, String) { ( self.record.resources_digest.clone(), self.record.receipt_digest.clone(), ) } + #[cfg(any(target_os = "linux", target_os = "macos"))] pub(super) fn into_retirement_file(self) -> LockedRecord { self.file } + #[cfg(any(target_os = "linux", target_os = "macos"))] pub(super) fn file_for_retirement(&self) -> &File { &self.file } @@ -126,6 +129,7 @@ impl Acknowledged { } } +#[cfg(any(target_os = "linux", target_os = "macos"))] pub(super) fn verify_retirement_context( file: &File, plan: &super::CheckoutRetirementPlan, diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index c0a50bd81..c42671208 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -14,8 +14,8 @@ The merge retains snapshot admission/reclamation and the incoming root-schema validation/settlement events, plus the binary-worker activity-drain wait. The seven pre-existing uncommitted files were restored from a retained backup stash. The branch still contains substantial undelivered recovery work; it must not be -confused with accepted main behavior. GitHub reported both issues open and no -open PRs at audit time. The issue #93 checklist trails the repository's merged +confused with accepted main behavior. Both issues remain open. Draft PR #123 now consolidates this branch for +review and full CI. The issue #93 checklist trails the repository's merged section 5/6 implementation state; reconcile it at delivery. | Area | Evidence and remaining work | @@ -94,6 +94,24 @@ warnings denied. Strict OpenSpec passed 100 items and lint reported zero errors with 25 warnings. Native C deprecation and linker unwind-size warnings were observed during compilation. Current-source native acceptance remains required. +### Consolidated PR and Windows core lint + +Draft PR [#123](https://github.com/CyberdyneCorp/cybercode-cli/pull/123) carries +the remaining goal implementation. Its first full CI revision `64052cba` failed +Windows binary-core Clippy because four acknowledgement-retirement helpers were +compiled without their Linux/macOS native consumers. The helpers now use the same +platform condition as those consumers. Shared acknowledgement verification stays +available to its existing callers. + +The binary proof workflow now also watches the worktree source subtree, which +contains these shared core dependencies. Windows verification of this correction +requires the new revision's CI; the local toolchain has only a macOS target. +Local core library/test Clippy passed with warnings denied, and 17 snapshot +acknowledgement/publication/retirement tests passed. Two ignored entries are +subprocess fixtures invoked by the process-kill parent tests. Workflow YAML, +formatting and diff checks passed. The other checks and full-goal acceptance +remain open. + ### Zero-byte ordinary restore continuation Ordinary cleanup now records payload completion for exact empty inventories and From 90234d37fdefc38f0137fef10eec58664b1d6d7a Mon Sep 17 00:00:00 2001 From: Edvane Cristina Santos Daltro Date: Tue, 10 Nov 2026 23:43:29 -0400 Subject: [PATCH 87/87] Gate Unix Git corruption fixture helpers on Unix --- crates/cyber-snapshot/tests/snapshots.rs | 2 ++ docs/goal-delivery.md | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/crates/cyber-snapshot/tests/snapshots.rs b/crates/cyber-snapshot/tests/snapshots.rs index 262fba07c..5ade563ae 100644 --- a/crates/cyber-snapshot/tests/snapshots.rs +++ b/crates/cyber-snapshot/tests/snapshots.rs @@ -32,6 +32,7 @@ fn git(dir: &Path, args: &[&str]) -> String { } impl Repo { + #[cfg(unix)] fn shadow_directory(&self) -> PathBuf { std::fs::read_dir(self.data.join("snapshot")) .unwrap() @@ -40,6 +41,7 @@ impl Repo { .find(|path| path.join("HEAD").exists()) .unwrap() } + #[cfg(unix)] fn shadow_object(&self, revision: &str) -> PathBuf { let shadow = self.shadow_directory(); // v3 tags peel to a retention envelope. Corrupt the canonical graph diff --git a/docs/goal-delivery.md b/docs/goal-delivery.md index c42671208..3e772929b 100644 --- a/docs/goal-delivery.md +++ b/docs/goal-delivery.md @@ -112,6 +112,17 @@ subprocess fixtures invoked by the process-kill parent tests. Workflow YAML, formatting and diff checks passed. The other checks and full-goal acceptance remain open. +### Windows snapshot fixture lint + +The Windows binary proof on `2a628f12` passed core Clippy and all 25 selected +binary tests without skips. The broader Windows workspace lint then failed on +`Repo::shadow_directory` and `Repo::shadow_object` in the Git snapshot test +fixture. Their corruption-test callers are Unix-only, so the helpers now use the +same condition. Local snapshot all-target Clippy passed with warnings denied. +All 22 Git snapshot integration tests passed; the ignored entry is the pathspec +subprocess fixture invoked by its parent test. Formatting and diff checks passed. +Current-head Windows lint must verify this separate correction. + ### Zero-byte ordinary restore continuation Ordinary cleanup now records payload completion for exact empty inventories and