From 505a24d11e856f6c52d0b050b1d56a205a7a00fc Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 04:54:46 -0700 Subject: [PATCH 01/64] feat: route executable Actions through scheduler Ticks --- .github/workflows/ci.yml | 4 + .github/workflows/macos-local.yml | 4 + crates/warp-core/Cargo.toml | 2 +- crates/warp-core/src/causal_wal.rs | 60 +- crates/warp-core/src/causal_wal_tests.rs | 38 +- crates/warp-core/src/contract_obstruction.rs | 5 + crates/warp-core/src/coordinator.rs | 282 +++++- crates/warp-core/src/echo_operation.rs | 644 +++++++++++- crates/warp-core/src/engine_impl.rs | 5 +- crates/warp-core/src/head_inbox.rs | 99 ++ crates/warp-core/src/lib.rs | 12 +- crates/warp-core/src/provenance_codec.rs | 6 + crates/warp-core/src/receipt.rs | 27 +- crates/warp-core/src/trusted_runtime_host.rs | 919 ++++++++++++++---- .../executable_operation_pipeline_tests.rs | 651 ++++++++++++- 15 files changed, 2551 insertions(+), 207 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8754f4b..4834b00e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,8 @@ jobs: run: | cargo clippy -p warp-core --features host_test --test causal_fact_publication_tests -- -D warnings -D missing_docs cargo clippy -p warp-core --features host_test --test optic_invocation_admission_tests -- -D warnings -D missing_docs + - name: cargo clippy (scheduler-owned executable Actions) + run: cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs clippy-det-fixed: name: Clippy (det_fixed) @@ -301,6 +303,8 @@ jobs: run: cargo test -p warp-core --features native_rule_bootstrap,host_test --test installed_contract_intent_pipeline_tests - name: cargo test (warp-core provider proposal admission) run: cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime --test provider_contract_admission_tests + - name: cargo test (scheduler-owned executable Actions) + run: cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests - name: cargo test --doc (warp-core) run: cargo test -p warp-core --doc - name: PRNG golden regression (warp-math) diff --git a/.github/workflows/macos-local.yml b/.github/workflows/macos-local.yml index 5a1d3810..62b7f3ec 100644 --- a/.github/workflows/macos-local.yml +++ b/.github/workflows/macos-local.yml @@ -51,6 +51,8 @@ jobs: run: | cargo clippy -p warp-core --features host_test --test causal_fact_publication_tests -- -D warnings -D missing_docs cargo clippy -p warp-core --features host_test --test optic_invocation_admission_tests -- -D warnings -D missing_docs + - name: cargo clippy (scheduler-owned executable Actions) + run: cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs - name: cargo test (workspace) run: cargo test --workspace - name: cargo test (warp-core host_test admission fixtures) @@ -59,3 +61,5 @@ jobs: cargo test -p warp-core --features host_test --test optic_invocation_admission_tests - name: cargo test (warp-core installed contract intent pipeline) run: cargo test -p warp-core --features native_rule_bootstrap,host_test --test installed_contract_intent_pipeline_tests + - name: cargo test (scheduler-owned executable Actions) + run: cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests diff --git a/crates/warp-core/Cargo.toml b/crates/warp-core/Cargo.toml index 7504ab5b..20197b3d 100644 --- a/crates/warp-core/Cargo.toml +++ b/crates/warp-core/Cargo.toml @@ -95,7 +95,7 @@ required-features = ["native_rule_bootstrap", "trusted_runtime"] [[test]] name = "executable_operation_pipeline_tests" -required-features = ["native_rule_bootstrap", "trusted_runtime"] +required-features = ["native_rule_bootstrap", "trusted_runtime", "host_test"] [build-dependencies] blake3 = "1.0" diff --git a/crates/warp-core/src/causal_wal.rs b/crates/warp-core/src/causal_wal.rs index b6399c3c..f007b49d 100644 --- a/crates/warp-core/src/causal_wal.rs +++ b/crates/warp-core/src/causal_wal.rs @@ -446,6 +446,9 @@ pub enum WalRecordKind { ExecutableOperationExecutionRecorded, /// Echo execution kernel retained the outcome's replayable state delta. ExecutableOperationStateDeltaRecorded, + /// Trusted scheduler retained one typed executable-operation Action + /// outcome inside a scheduler-owned Tick. + ExecutableOperationActionOutcomeRecorded, } impl WalRecordKind { @@ -479,6 +482,9 @@ impl WalRecordKind { Self::ExecutableOperationPackageInstalled => "ExecutableOperationPackageInstalled", Self::ExecutableOperationExecutionRecorded => "ExecutableOperationExecutionRecorded", Self::ExecutableOperationStateDeltaRecorded => "ExecutableOperationStateDeltaRecorded", + Self::ExecutableOperationActionOutcomeRecorded => { + "ExecutableOperationActionOutcomeRecorded" + } } } @@ -494,6 +500,7 @@ impl WalRecordKind { | Self::TickReceiptRecorded | Self::RuntimeStateDeltaRecorded | Self::ReceiptCorrelationRecorded + | Self::ExecutableOperationActionOutcomeRecorded | Self::ReadingEnvelopeRetained | Self::RetainedMaterialRefRecorded | Self::MaterializationIntentRecorded @@ -558,6 +565,7 @@ impl WalRecordKind { Self::ExecutableOperationPackageInstalled => 25, Self::ExecutableOperationExecutionRecorded => 26, Self::ExecutableOperationStateDeltaRecorded => 27, + Self::ExecutableOperationActionOutcomeRecorded => 28, } } @@ -590,6 +598,7 @@ impl WalRecordKind { 25 => Ok(Self::ExecutableOperationPackageInstalled), 26 => Ok(Self::ExecutableOperationExecutionRecorded), 27 => Ok(Self::ExecutableOperationStateDeltaRecorded), + 28 => Ok(Self::ExecutableOperationActionOutcomeRecorded), _ => Err(WalDecodeError::UnknownEnumCode { enum_name: "WalRecordKind", code, @@ -2436,7 +2445,7 @@ impl WalTickDecision { /// Returns `true` when this decision is a lawful rejection, not a fault. #[must_use] pub const fn is_lawful_rejection(self) -> bool { - matches!(self, Self::RejectedFootprintConflict) + matches!(self, Self::RejectedFootprintConflict | Self::Obstructed) } } @@ -7771,18 +7780,58 @@ pub fn build_tick_transaction( /// Builds a scheduler-owned tick transaction with replayable state-delta material. pub fn build_replayable_tick_transaction( - mut builder: WalTransactionBuilder, + builder: WalTransactionBuilder, receipt: TickReceiptRecord, correlation: WalReceiptCorrelationRecord, retained_state_delta_bytes: Vec, affected_frontiers: Vec, ) -> Result { + build_replayable_tick_batch_transaction( + builder, + vec![(receipt, correlation, None)], + retained_state_delta_bytes, + affected_frontiers, + ) +} + +/// Builds one scheduler-owned Tick transaction with one or more per-Action +/// receipt correlations and exactly one replayable state consequence. +pub(crate) fn build_replayable_tick_batch_transaction( + mut builder: WalTransactionBuilder, + receipts: Vec<( + TickReceiptRecord, + WalReceiptCorrelationRecord, + Option>, + )>, + retained_state_delta_bytes: Vec, + affected_frontiers: Vec, +) -> Result { + let action_outcome_count = receipts + .iter() + .filter(|(_, _, action_outcome)| action_outcome.is_some()) + .count(); + if action_outcome_count != 0 && action_outcome_count != receipts.len() { + return Err(WalBuildError::TickBatchActionOutcomeShapeMismatch); + } let state_delta = WalRuntimeStateDeltaRecord::from_payload_bytes(&retained_state_delta_bytes) .map_err(|_| WalBuildError::RuntimeStateDeltaInvalid)?; - if state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest { + if receipts.is_empty() + || receipts.iter().any(|(receipt, correlation, _)| { + state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest + || receipt.receipt_ref != correlation.receipt_ref + }) + { return Err(WalBuildError::RuntimeStateDeltaReceiptMismatch); } - push_tick_receipt_records(&mut builder, receipt, &correlation)?; + for (receipt, correlation, action_outcome) in receipts { + push_tick_receipt_records(&mut builder, receipt, &correlation)?; + if let Some(action_outcome) = action_outcome { + builder.push_record( + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + action_outcome, + )?; + } + } builder.push_record( WalRecordKind::RuntimeStateDeltaRecorded, retained_state_delta_bytes, @@ -8852,6 +8901,9 @@ pub enum WalBuildError { /// Replayable runtime state delta names a different receipt commitment. #[error("WAL replayable runtime state delta does not match the tick receipt")] RuntimeStateDeltaReceiptMismatch, + /// A scheduler Tick batch mixes Action and non-Action receipt members. + #[error("WAL scheduler Tick batch has inconsistent Action outcome framing")] + TickBatchActionOutcomeShapeMismatch, /// Validation failed. #[error(transparent)] Validation(#[from] WalValidationError), diff --git a/crates/warp-core/src/causal_wal_tests.rs b/crates/warp-core/src/causal_wal_tests.rs index 70806e49..78332947 100644 --- a/crates/warp-core/src/causal_wal_tests.rs +++ b/crates/warp-core/src/causal_wal_tests.rs @@ -14,9 +14,9 @@ use crate::causal_wal::{ apply_committed_transaction, audit_wal_release_readiness, build_causal_anchor_admission_transaction, build_checkpoint_publication_transaction, build_materialization_outbox_transaction, build_recovery_certificate, - build_retained_reading_transaction, build_submission_acceptance_transaction, - build_submission_acceptance_with_material_transaction, build_tick_transaction, - build_topology_intent_transaction, canonical_segment_relative_path, + build_replayable_tick_batch_transaction, build_retained_reading_transaction, + build_submission_acceptance_transaction, build_submission_acceptance_with_material_transaction, + build_tick_transaction, build_topology_intent_transaction, canonical_segment_relative_path, causal_anchor_frontier_digest_from_evidence, causal_anchor_genesis_frontier_digest, causal_history_genesis_frontier_digest, doctor_in_memory_store, evaluate_checkpoint_publication, lint_wal_schema_terms, logical_causal_history_frontier_digest, @@ -3184,6 +3184,37 @@ fn tick_transaction_rejects_mismatched_receipt_correlation() { assert_eq!(error, WalBuildError::ReceiptCorrelationMismatch); } +#[test] +fn tick_batch_rejects_mixed_action_outcome_framing() { + let error = must_err( + build_replayable_tick_batch_transaction( + builder( + transaction_id("tx:tick:mixed-action-outcomes"), + Lsn::from_raw(0), + WalAppendAuthority::TrustedScheduler, + WalTransactionKind::SchedulerTick, + ), + vec![ + ( + receipt_record("action", WalTickDecision::Applied), + correlation_record("action"), + Some(b"retained-action-outcome".to_vec()), + ), + ( + receipt_record("legacy", WalTickDecision::Applied), + correlation_record("legacy"), + None, + ), + ], + b"state-delta-is-not-decoded-before-shape-validation".to_vec(), + Vec::new(), + ), + "one scheduler Tick must not mix Action and non-Action receipt members", + ); + + assert_eq!(error, WalBuildError::TickBatchActionOutcomeShapeMismatch); +} + #[test] fn receipt_correlation_decode_rejects_noncanonical_parent_sets() { let parent_a = causal_receipt_ref("parent-a"); @@ -3716,6 +3747,7 @@ fn lawful_rejection_recovers_without_fault_posture() { .copied() ) .is_lawful_rejection()); + assert!(WalTickDecision::Obstructed.is_lawful_rejection()); } #[test] diff --git a/crates/warp-core/src/contract_obstruction.rs b/crates/warp-core/src/contract_obstruction.rs index cda0806f..206c23b8 100644 --- a/crates/warp-core/src/contract_obstruction.rs +++ b/crates/warp-core/src/contract_obstruction.rs @@ -228,6 +228,11 @@ impl ContractObstruction { submission_id: *submission_id, }, ), + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationCommit(_) + | RuntimeError::EchoOperationActionAdmissionMissing(_) => { + Self::runtime_fault(ContractObstructionSubject::Unspecified) + } RuntimeError::SchedulerFaultGenerationOverflow | RuntimeError::DuplicateWorldline(_) | RuntimeError::DuplicateHead(_) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 411ff112..925155ff 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -11,6 +11,13 @@ use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; use thiserror::Error; use crate::clock::{GlobalTick, WorldlineTick}; +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +use crate::echo_operation::{ + AdmittedEchoOperationInvocationV1, EchoOperationActionOutcomeV1, + EchoOperationApplicationBasisV1, EchoOperationCommitErrorV1, + EchoOperationEvaluationAuthorityV1, EchoOperationEvaluationBasisV1, + SchedulerEchoOperationCandidateV1, +}; use crate::engine_impl::{CommitOutcome, Engine, EngineError}; use crate::head::{ HeadEligibility, PlaybackHeadRegistry, RunnableWriterSet, WriterHead, WriterHeadKey, @@ -40,6 +47,49 @@ use crate::CausalTickReceiptRef; #[cfg(feature = "native_rule_bootstrap")] const INSTALLED_CONTRACT_EINT_INTENT_KIND_LABEL: &str = "echo.intent/eint-v1"; +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +type SchedulerOperationOutcomeV1 = (Hash, EchoOperationActionOutcomeV1); +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +type SchedulerOperationOutcomesV1 = Vec; +#[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] +type SchedulerOperationOutcomeV1 = (); +#[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] +type SchedulerOperationOutcomesV1 = Vec; + +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +pub(crate) fn resolve_echo_operation_evaluation_basis_v1( + runtime: &WorldlineRuntime, + provenance: &ProvenanceService, + writer_head: WriterHeadKey, + application_basis: EchoOperationApplicationBasisV1, +) -> Result { + if runtime.heads().get(&writer_head).is_none() { + return Err(RuntimeError::UnknownHead(writer_head)); + } + let frontier = runtime + .worldlines() + .get(&writer_head.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(writer_head.worldline_id))?; + let state_root = frontier.state().state_root(); + let (commit_global_tick, commit_id) = match provenance.tip_ref(writer_head.worldline_id)? { + Some(tip) => { + let entry = provenance.entry(tip.worldline_id, tip.worldline_tick)?; + (Some(entry.commit_global_tick), entry.expected.commit_hash) + } + None => ( + None, + crate::echo_operation::genesis_commit_id(writer_head, state_root), + ), + }; + Ok(EchoOperationEvaluationBasisV1::new( + writer_head, + frontier.frontier_tick(), + commit_global_tick, + state_root, + commit_id, + application_basis, + )) +} // ============================================================================= // Runtime Errors and Ingress Disposition @@ -91,6 +141,16 @@ pub enum RuntimeError { /// A commit against a worldline frontier failed. #[error(transparent)] Engine(#[from] EngineError), + /// A scheduler-owned executable-operation Action batch could not be + /// constructed. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[error(transparent)] + EchoOperationCommit(#[from] EchoOperationCommitErrorV1), + /// The scheduler selected a reserved executable Action without the + /// runtime-owned admission token that authorizes evaluation. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[error("executable-operation Action admission is unavailable for ingress {0:?}")] + EchoOperationActionAdmissionMissing(Hash), /// Provenance append or lookup failed during a runtime step. #[error(transparent)] Provenance(#[from] HistoryError), @@ -1116,6 +1176,12 @@ pub struct WorldlineRuntime { next_scheduler_fault_generation: SchedulerFaultGeneration, /// Registry of live speculative strands attached to the runtime. strands: StrandRegistry, + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + fail_next_echo_operation_action_tick_construction: bool, } #[derive(Clone, Debug)] @@ -1268,6 +1334,24 @@ impl WorldlineRuntime { &self.worldlines } + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + pub(crate) fn inject_echo_operation_action_tick_construction_failure_for_test(&mut self) { + self.fail_next_echo_operation_action_tick_construction = true; + } + + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + fn take_echo_operation_action_tick_construction_failure_for_test(&mut self) -> bool { + std::mem::take(&mut self.fail_next_echo_operation_action_tick_construction) + } + /// Returns the registered writer heads. #[must_use] pub fn heads(&self) -> &PlaybackHeadRegistry { @@ -1649,11 +1733,36 @@ impl WorldlineRuntime { return no_match(); } candidate + } else if let Some(candidate) = candidates.next() { + if candidates.next().is_some() { + return no_match(); + } + candidate } else { - let Some(candidate) = candidates.next() else { + let current_basis = receipt_correlation_current_basis(correlation); + let Some(ticketed_ingress_ids) = self + .receipt_correlations_by_current_basis + .get(¤t_basis) + else { return no_match(); }; - candidate + let mut same_tick = ticketed_ingress_ids + .iter() + .filter_map(|ticketed_ingress_id| { + self.receipt_correlations_by_ticketed_ingress + .get(ticketed_ingress_id) + }) + .collect::>(); + same_tick.sort_by_key(|candidate| candidate.ingress_id); + let Some(index) = same_tick.iter().position(|candidate| { + candidate.ticketed_ingress_id == correlation.ticketed_ingress_id + }) else { + return no_match(); + }; + let Some(entry) = receipt.entries().get(index) else { + return no_match(); + }; + (index, entry) }; let (idx, entry) = candidate; let Ok(receipt_entry_index) = u32::try_from(idx) else { @@ -2207,6 +2316,7 @@ impl WorldlineRuntime { .entries() .iter() .any(|entry| entry.scope.local_id == ingress_node) + && crate::echo_operation::echo_operation_action_invocation_bytes_v1(envelope).is_none() { return Err(RuntimeError::ReceiptCorrelationReplayMismatch( persisted.causal_receipt_ref.identity_digest(), @@ -2437,6 +2547,23 @@ impl WorldlineRuntime { self.ingest_ticketed_invocation_inner(submission_id, ticket.ticket_digest, envelope, None) } + /// Stages one runtime-admitted executable-operation Action into the normal + /// head inbox. + /// + /// The opaque admission digest is derived by the trusted runtime owner from + /// exact installed meaning and invocation-admission evidence. No contract + /// callback evidence is attached to this ingress category. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn ingest_echo_operation_action_v1( + &mut self, + _authority: &TicketedRuntimeIngressAuthority, + submission_id: Hash, + admission_digest: Hash, + envelope: IngressEnvelope, + ) -> Result { + self.ingest_ticketed_invocation_inner(submission_id, admission_digest, envelope, None) + } + fn ingest_ticketed_invocation_inner( &mut self, submission_id: Hash, @@ -3042,6 +3169,8 @@ fn scheduler_fault_scope_for_error( RuntimeError::Engine(_) | RuntimeError::FrontierTickOverflow(_) => { SchedulerFaultScope::Head(head_key) } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationCommit(_) => SchedulerFaultScope::Head(head_key), RuntimeError::Provenance(_) | RuntimeError::UnknownHead(_) | RuntimeError::UnknownWorldline(_) @@ -3074,6 +3203,8 @@ fn scheduler_fault_scope_for_error( | RuntimeError::TicketedIngressDuplicateRuntimeIngress { .. } => { SchedulerFaultScope::Runtime } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationActionAdmissionMissing(_) => SchedulerFaultScope::Runtime, } } @@ -3714,6 +3845,16 @@ fn scheduler_error_cause_digest(err: &RuntimeError) -> Hash { } } } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationCommit(error) => { + hasher.update(b"echo-operation-commit"); + hasher.update(error.to_string().as_bytes()); + } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationActionAdmissionMissing(ingress_id) => { + hasher.update(b"echo-operation-action-admission-missing"); + hasher.update(ingress_id); + } RuntimeError::FrontierTickOverflow(worldline_id) => { hasher.update(b"frontier-tick-overflow"); hasher.update(worldline_id.as_bytes()); @@ -3917,12 +4058,50 @@ impl SchedulerCoordinator { provenance: &mut ProvenanceService, engine: &mut Engine, ) -> Result, RuntimeError> { + Self::super_tick_inner(runtime, provenance, engine, None) + .map(|(records, _operation_outcomes)| records) + } + + /// Executes one scheduler pass with runtime-admitted executable-operation + /// Actions available to Tick construction. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn super_tick_with_echo_operation_actions_v1( + runtime: &mut WorldlineRuntime, + provenance: &mut ProvenanceService, + engine: &mut Engine, + operation_actions: &BTreeMap, + evaluation_authority: &EchoOperationEvaluationAuthorityV1, + ) -> Result<(Vec, SchedulerOperationOutcomesV1), RuntimeError> { + Self::super_tick_inner( + runtime, + provenance, + engine, + Some((operation_actions, evaluation_authority)), + ) + } + + fn super_tick_inner( + runtime: &mut WorldlineRuntime, + provenance: &mut ProvenanceService, + engine: &mut Engine, + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + operation_actions: Option<( + &BTreeMap, + &EchoOperationEvaluationAuthorityV1, + )>, + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + _operation_actions: Option<()>, + ) -> Result<(Vec, SchedulerOperationOutcomesV1), RuntimeError> { if let Some(fault_id) = runtime.runtime_fault { return Err(RuntimeError::SchedulerRuntimeFaultActive(fault_id)); } runtime.refresh_runnable(); let mut records = Vec::new(); + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let mut operation_outcomes = Vec::new(); + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + let operation_outcomes = Vec::new(); let keys: Vec = runtime.runnable.iter().copied().collect(); let next_global_tick = if let Some(next) = runtime.global_tick.checked_increment() { next @@ -3961,11 +4140,20 @@ impl SchedulerCoordinator { provenance.checkpoint_for(keys.iter().map(|key| key.worldline_id))?; for key in &keys { - let admitted = runtime + let inbox = runtime .heads .inbox_mut(key) - .ok_or(RuntimeError::UnknownHead(*key))? - .admit(); + .ok_or(RuntimeError::UnknownHead(*key))?; + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let admitted = if operation_actions.is_some() { + inbox.admit_partitioned( + crate::echo_operation::echo_operation_action_intent_kind_v1(), + ) + } else { + inbox.admit() + }; + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + let admitted = inbox.admit(); if admitted.is_empty() { continue; @@ -3979,6 +4167,88 @@ impl SchedulerCoordinator { .frontier_tick(); let parents = provenance.tip_ref(key.worldline_id)?.into_iter().collect(); + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let executable_action_batch = admitted.first().is_some_and(|envelope| { + crate::echo_operation::echo_operation_action_invocation_bytes_v1(envelope) + .is_some() + }); + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let (snapshot, patch, receipt) = if executable_action_batch { + #[cfg(any(test, feature = "host_test"))] + if runtime.take_echo_operation_action_tick_construction_failure_for_test() { + return Err(RuntimeError::EchoOperationCommit( + EchoOperationCommitErrorV1::InjectedTickConstructionFailure, + )); + } + let Some((admitted_actions, evaluation_authority)) = operation_actions else { + return Err(RuntimeError::EchoOperationActionAdmissionMissing( + admitted[0].ingress_id(), + )); + }; + let mut candidates = Vec::with_capacity(admitted.len()); + for envelope in &admitted { + let ingress_id = envelope.ingress_id(); + let admitted_action = admitted_actions.get(&ingress_id).ok_or( + RuntimeError::EchoOperationActionAdmissionMissing(ingress_id), + )?; + let current_basis = resolve_echo_operation_evaluation_basis_v1( + runtime, + provenance, + *key, + admitted_action.evaluation_basis().application_basis(), + )?; + let state = runtime + .worldlines + .get(&key.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))? + .state(); + let preparation = crate::echo_operation::prepare_operation_v1( + engine + .installed_echo_operation_package_v1(admitted_action.package_id()), + admitted_action.clone(), + current_basis, + state, + engine.echo_operation_policy_id(), + evaluation_authority, + ); + candidates.push(SchedulerEchoOperationCandidateV1 { + ingress_id, + scope: admitted_action.scope(), + rule_id: admitted_action.installed_operation_id().as_hash(), + preparation, + }); + } + let material = { + let frontier = runtime + .worldlines + .frontier_mut(&key.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))?; + crate::echo_operation::commit_scheduler_action_batch_to_state_v1( + candidates, + frontier.state_mut(), + next_global_tick, + engine.echo_operation_policy_id(), + )? + }; + operation_outcomes.extend(material.outcomes); + (material.snapshot, material.patch, material.tick_receipt) + } else { + let CommitOutcome { + snapshot, + patch, + receipt, + } = { + let frontier = runtime + .worldlines + .frontier_mut(&key.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))?; + engine + .commit_with_state(frontier.state_mut(), &admitted) + .map_err(RuntimeError::from)? + }; + (snapshot, patch, receipt) + }; + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] let CommitOutcome { snapshot, patch, @@ -4100,7 +4370,7 @@ impl SchedulerCoordinator { } runtime.global_tick = next_global_tick; - Ok(records) + Ok((records, operation_outcomes)) } /// Returns the canonical ordering of runnable heads without mutating state. diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 9d3d69bd..a5a29fb2 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -33,8 +33,9 @@ use crate::{ clock::{GlobalTick, WorldlineTick}, footprint::{Footprint, WarpScopedPortKey}, head::WriterHeadKey, + head_inbox::{make_intent_kind, IngressEnvelope, IngressPayload, IngressTarget, IntentKind}, ident::{EdgeKey, Hash, NodeKey, TypeId}, - receipt::{TickReceipt, TickReceiptDisposition, TickReceiptEntry}, + receipt::{TickReceipt, TickReceiptDisposition, TickReceiptEntry, TickReceiptRejection}, record::NodeRecord, snapshot::{compute_commit_hash_v2, Snapshot}, tick_patch::{SlotId, TickCommitStatus, TickPatchError, WarpOp, WarpTickPatchV1}, @@ -78,6 +79,12 @@ const INTRINSIC_PROFILE: &str = "echo.operation-attachment-algebra/v1"; const PACKAGE_ID_DOMAIN: &[u8] = b"echo:operation-package:v1\0"; const PROGRAM_ID_DOMAIN: &[u8] = b"echo:operation-program:v1\0"; const INVOCATION_ID_DOMAIN: &[u8] = b"echo:operation-invocation:v1\0"; +const ACTION_INTENT_KIND: &str = "echo.executable-operation-action/v1"; +const ACTION_BATCH_RULE_PACK_DOMAIN: &[u8] = b"echo:operation-action-batch-rule-pack:v1\0"; +const ACTION_BATCH_PLAN_DOMAIN: &[u8] = b"echo:operation-action-batch-plan:v1\0"; +const ACTION_BATCH_REWRITES_DOMAIN: &[u8] = b"echo:operation-action-batch-rewrites:v1\0"; +const ACTION_BATCH_COMPOSITION_DOMAIN: &[u8] = b"echo:operation-action-batch-composition:v1\0"; +const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT001"; const INVOCATION_BYTES_DIGEST_DOMAIN: &[u8] = b"echo:operation-invocation-bytes:v1\0"; const BASIS_ID_DOMAIN: &[u8] = b"echo:operation-evaluation-basis:v1\0"; const PACKAGE_ADMISSION_ID_DOMAIN: &[u8] = b"echo:operation-package-admission:v1\0"; @@ -270,6 +277,49 @@ pub fn echo_operation_package_id_v1(bytes: &[u8]) -> EchoOperationPackageIdV1 { EchoOperationPackageIdV1(domain_hash(PACKAGE_ID_DOMAIN, bytes)) } +/// Returns the reserved stable kind for Echo-interpreted executable-operation +/// Actions. +#[must_use] +pub fn echo_operation_action_intent_kind_v1() -> IntentKind { + make_intent_kind(ACTION_INTENT_KIND) +} + +/// Wraps one canonical executable-operation invocation in the ordinary Echo +/// Action envelope. +/// +/// The helper decodes and reproduces the invocation before constructing +/// ingress, so non-canonical or unsupported bytes never acquire the reserved +/// Action kind. +pub fn echo_operation_action_envelope_v1( + target: IngressTarget, + canonical_invocation_bytes: Vec, +) -> Result { + let invocation = EchoOperationInvocationV1::from_canonical_bytes(&canonical_invocation_bytes)?; + if invocation.to_canonical_bytes()? != canonical_invocation_bytes { + return Err(artifact_error( + EchoOperationArtifactErrorKindV1::NonCanonical, + "executable-operation Action must carry exact canonical invocation bytes", + )); + } + Ok(IngressEnvelope::local_intent( + target, + echo_operation_action_intent_kind_v1(), + canonical_invocation_bytes, + )) +} + +pub(crate) fn echo_operation_action_invocation_bytes_v1( + envelope: &IngressEnvelope, +) -> Option<&[u8]> { + match envelope.payload() { + IngressPayload::LocalIntent { + intent_kind, + intent_bytes, + } if *intent_kind == echo_operation_action_intent_kind_v1() => Some(intent_bytes), + IngressPayload::LocalIntent { .. } => None, + } +} + /// Computes the digest of one typed attachment atom. #[must_use] pub fn echo_operation_atom_value_digest_v1(type_id: TypeId, bytes: &[u8]) -> Hash { @@ -2235,6 +2285,18 @@ impl AdmittedEchoOperationInvocationV1 { pub(crate) const fn evaluation_basis(&self) -> EchoOperationEvaluationBasisV1 { self.invocation.evaluation_basis } + + pub(crate) const fn scope(&self) -> NodeKey { + self.invocation.node + } + + pub(crate) const fn installed_operation_id(&self) -> InstalledEchoOperationIdV1 { + self.installed_operation_id + } + + pub(crate) const fn admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { + self.admission_id + } } pub(crate) fn admit_invocation_v1( @@ -2429,6 +2491,39 @@ pub(crate) fn decode_invocation_route_v1( Ok((invocation.package_id, invocation.evaluation_basis)) } +pub(crate) struct EchoOperationActionInvocationEvidenceV1 { + pub package_id: EchoOperationPackageIdV1, + pub scope: NodeKey, + pub evaluation_basis: EchoOperationEvaluationBasisV1, + pub invocation_id: EchoOperationInvocationIdV1, + pub invocation_bytes_digest: Hash, +} + +pub(crate) fn inspect_action_invocation_v1( + canonical_invocation_bytes: &[u8], +) -> Result { + let invocation = EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) + .map_err(|error| { + invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::MalformedInvocation, + error.to_string(), + ) + })?; + Ok(EchoOperationActionInvocationEvidenceV1 { + package_id: invocation.package_id, + scope: invocation.node, + evaluation_basis: invocation.evaluation_basis, + invocation_id: EchoOperationInvocationIdV1(domain_hash( + INVOCATION_ID_DOMAIN, + canonical_invocation_bytes, + )), + invocation_bytes_digest: domain_hash( + INVOCATION_BYTES_DIGEST_DOMAIN, + canonical_invocation_bytes, + ), + }) +} + pub(crate) fn invocation_runtime_error( detail: impl Into, ) -> EchoOperationInvocationAdmissionErrorV1 { @@ -2500,12 +2595,30 @@ impl EchoOperationObstructionV1 { self.kind } + /// Returns the installed package whose evaluation was obstructed. + #[must_use] + pub const fn package_id(&self) -> EchoOperationPackageIdV1 { + self.package_id + } + /// Returns the exact installed operation against which evaluation was attempted. #[must_use] pub const fn installed_operation_id(&self) -> InstalledEchoOperationIdV1 { self.installed_operation_id } + /// Returns the exact canonical invocation refused by evaluation. + #[must_use] + pub const fn invocation_id(&self) -> EchoOperationInvocationIdV1 { + self.invocation_id + } + + /// Returns the exact private-evaluation basis refused by evaluation. + #[must_use] + pub const fn evaluation_basis_id(&self) -> EchoOperationEvaluationBasisIdV1 { + self.evaluation_basis_id + } + /// Returns the exact runtime admission that authorized this evaluation attempt. #[must_use] pub const fn invocation_admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { @@ -2527,6 +2640,233 @@ impl EchoOperationObstructionV1 { } } +/// Typed terminal outcome for one executable-operation Action selected by a +/// scheduler-owned Tick. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EchoOperationActionOutcomeV1 { + /// The Action's prepared member consequence entered the composite Tick. + Committed(Box), + /// Private bounded evaluation refused the Action and emitted no mutation. + Obstructed(EchoOperationObstructionV1), + /// An earlier applied Action in the same Tick reserved an overlapping + /// footprint. + RejectedFootprintConflict { + /// Exact installed operation whose prepared consequence conflicted. + installed_operation_id: InstalledEchoOperationIdV1, + /// Exact canonical invocation whose prepared consequence conflicted. + invocation_id: EchoOperationInvocationIdV1, + /// Identity of the privately prepared candidate that was not applied. + preparation_id: PreparedEchoOperationIdV1, + /// Canonical indices of earlier applied Tick members that blocked it. + blocked_by: Vec, + }, +} + +pub(crate) fn retain_action_outcome_v1( + submission_id: Hash, + ingress_id: Hash, + outcome: &EchoOperationActionOutcomeV1, +) -> Result, EchoOperationArtifactErrorV1> { + let mut out = Vec::new(); + out.extend_from_slice(ACTION_OUTCOME_RECORD_MAGIC); + out.extend_from_slice(&submission_id); + out.extend_from_slice(&ingress_id); + match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + out.push(1); + let receipt_bytes = receipt.to_canonical_bytes()?; + let receipt_len = u64::try_from(receipt_bytes.len()) + .map_err(|_| invalid_structure("Action receipt length is not representable"))?; + out.extend_from_slice(&receipt_len.to_le_bytes()); + out.extend_from_slice(&receipt_bytes); + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + out.push(2); + out.push(obstruction_kind_code(obstruction.kind)); + out.extend_from_slice(&obstruction.package_id.as_hash()); + out.extend_from_slice(&obstruction.installed_operation_id.as_hash()); + out.extend_from_slice(&obstruction.invocation_admission_id.as_hash()); + out.extend_from_slice(&obstruction.invocation_id.as_hash()); + out.extend_from_slice(&obstruction.evaluation_basis_id.as_hash()); + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict { + installed_operation_id, + invocation_id, + preparation_id, + blocked_by, + } => { + out.push(3); + out.extend_from_slice(&installed_operation_id.as_hash()); + out.extend_from_slice(&invocation_id.as_hash()); + out.extend_from_slice(&preparation_id.as_hash()); + let blocker_count = u64::try_from(blocked_by.len()) + .map_err(|_| invalid_structure("Action blocker count is not representable"))?; + out.extend_from_slice(&blocker_count.to_le_bytes()); + for blocker in blocked_by { + out.extend_from_slice(&blocker.to_le_bytes()); + } + } + } + Ok(out) +} + +pub(crate) fn recover_action_outcome_v1( + bytes: &[u8], +) -> Result<(Hash, Hash, EchoOperationActionOutcomeV1), EchoOperationArtifactErrorV1> { + let minimum = ACTION_OUTCOME_RECORD_MAGIC.len() + 32 + 32 + 1; + if bytes.len() < minimum + || &bytes[..ACTION_OUTCOME_RECORD_MAGIC.len()] != ACTION_OUTCOME_RECORD_MAGIC + { + return Err(invalid_structure( + "executable-operation Action outcome record has invalid framing", + )); + } + let mut offset = ACTION_OUTCOME_RECORD_MAGIC.len(); + let submission_id = read_action_outcome_hash(bytes, &mut offset)?; + let ingress_id = read_action_outcome_hash(bytes, &mut offset)?; + let tag = *bytes + .get(offset) + .ok_or_else(|| invalid_structure("Action outcome tag is missing"))?; + offset += 1; + let outcome = match tag { + 1 => { + let len = read_action_outcome_u64(bytes, &mut offset)?; + let len = usize::try_from(len) + .map_err(|_| invalid_structure("Action receipt length is not representable"))?; + let end = offset + .checked_add(len) + .ok_or_else(|| invalid_structure("Action receipt length overflows"))?; + let receipt_bytes = bytes + .get(offset..end) + .ok_or_else(|| invalid_structure("Action receipt bytes are truncated"))?; + offset = end; + EchoOperationActionOutcomeV1::Committed(Box::new( + EchoOperationReceiptV1::from_canonical_bytes(receipt_bytes)?, + )) + } + 2 => { + let kind_code = *bytes + .get(offset) + .ok_or_else(|| invalid_structure("Action obstruction kind is missing"))?; + offset += 1; + EchoOperationActionOutcomeV1::Obstructed(EchoOperationObstructionV1 { + kind: obstruction_kind_from_code(kind_code)?, + package_id: EchoOperationPackageIdV1(read_action_outcome_hash(bytes, &mut offset)?), + installed_operation_id: InstalledEchoOperationIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + invocation_admission_id: EchoOperationInvocationAdmissionIdV1( + read_action_outcome_hash(bytes, &mut offset)?, + ), + invocation_id: EchoOperationInvocationIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + evaluation_basis_id: EchoOperationEvaluationBasisIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + }) + } + 3 => { + let installed_operation_id = + InstalledEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let invocation_id = + EchoOperationInvocationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let preparation_id = + PreparedEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let blocker_count = read_action_outcome_u64(bytes, &mut offset)?; + let blocker_count = usize::try_from(blocker_count) + .map_err(|_| invalid_structure("Action blocker count is not representable"))?; + let mut blocked_by = Vec::with_capacity(blocker_count); + for _ in 0..blocker_count { + let end = offset + .checked_add(4) + .ok_or_else(|| invalid_structure("Action blocker offset overflows"))?; + let raw: [u8; 4] = bytes + .get(offset..end) + .ok_or_else(|| invalid_structure("Action blocker bytes are truncated"))? + .try_into() + .map_err(|_| invalid_structure("Action blocker is not four bytes"))?; + offset = end; + blocked_by.push(u32::from_le_bytes(raw)); + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict { + installed_operation_id, + invocation_id, + preparation_id, + blocked_by, + } + } + _ => { + return Err(invalid_structure( + "unknown executable-operation Action outcome tag", + )) + } + }; + if offset != bytes.len() { + return Err(invalid_structure( + "executable-operation Action outcome record has trailing bytes", + )); + } + Ok((submission_id, ingress_id, outcome)) +} + +fn read_action_outcome_hash( + bytes: &[u8], + offset: &mut usize, +) -> Result { + let end = offset + .checked_add(32) + .ok_or_else(|| invalid_structure("Action outcome hash offset overflows"))?; + let value = bytes + .get(*offset..end) + .ok_or_else(|| invalid_structure("Action outcome hash is truncated"))? + .try_into() + .map_err(|_| invalid_structure("Action outcome hash is not 32 bytes"))?; + *offset = end; + Ok(value) +} + +fn read_action_outcome_u64( + bytes: &[u8], + offset: &mut usize, +) -> Result { + let end = offset + .checked_add(8) + .ok_or_else(|| invalid_structure("Action outcome integer offset overflows"))?; + let raw: [u8; 8] = bytes + .get(*offset..end) + .ok_or_else(|| invalid_structure("Action outcome integer is truncated"))? + .try_into() + .map_err(|_| invalid_structure("Action outcome integer is not eight bytes"))?; + *offset = end; + Ok(u64::from_le_bytes(raw)) +} + +fn obstruction_kind_from_code( + code: u8, +) -> Result { + match code { + 1 => Ok(EchoOperationObstructionKindV1::OperationUnavailable), + 2 => Ok(EchoOperationObstructionKindV1::BasisChanged), + 3 => Ok(EchoOperationObstructionKindV1::BudgetExceeded), + 4 => Ok(EchoOperationObstructionKindV1::NodeMissing), + 5 => Ok(EchoOperationObstructionKindV1::NodeTypeMismatch), + 6 => Ok(EchoOperationObstructionKindV1::AttachmentMissing), + 7 => Ok(EchoOperationObstructionKindV1::AttachmentNotAtom), + 8 => Ok(EchoOperationObstructionKindV1::AttachmentTypeMismatch), + 9 => Ok(EchoOperationObstructionKindV1::PreconditionMismatch), + 10 => Ok(EchoOperationObstructionKindV1::FootprintViolation), + 11 => Ok(EchoOperationObstructionKindV1::ReplacementTooLarge), + 12 => Ok(EchoOperationObstructionKindV1::EvaluationAuthorityMismatch), + _ => Err(invalid_structure( + "unknown executable-operation obstruction kind", + )), + } +} + /// Private evaluation result: exactly one complete preparation or one obstruction. #[derive(Clone, Debug)] pub enum EchoOperationPreparationV1 { @@ -2640,6 +2980,10 @@ impl PreparedEchoOperationV1 { self.installed.installed_operation_id } + pub(crate) const fn invocation_id(&self) -> EchoOperationInvocationIdV1 { + self.invocation_id + } + pub(crate) fn is_owned_by(&self, authority: &EchoOperationEvaluationAuthorityV1) -> bool { self.evaluation_authority.same_owner(authority) } @@ -3092,7 +3436,10 @@ impl EchoOperationReceiptV1 { self.prepared_patch_digest } - /// Returns singleton composition evidence only for a committed consequence. + /// Returns composition evidence only for a committed consequence. + /// + /// Direct compatibility commits bind singleton composition. Scheduler-owned + /// Action receipts bind the exact composite Tick consequence. #[must_use] pub const fn composition_digest(&self) -> Option { self.composition_digest @@ -3552,9 +3899,10 @@ impl EchoOperationReceiptV1 { || receipt.commit_id == [0; 32] || receipt.tick_receipt_digest == [0; 32] || receipt.committed_patch_digest.is_none() - || receipt.committed_patch_digest != Some(receipt.prepared_patch_digest) || receipt.committed_result_id != Some(receipt.prepared_result_id) - || receipt.composition_digest != Some(expected_composition_digest) + || receipt.composition_digest.is_none() + || (receipt.committed_patch_digest == Some(receipt.prepared_patch_digest) + && receipt.composition_digest != Some(expected_composition_digest)) || receipt.state_root_before != receipt.evaluation_basis.state_root || committed_worldline_tick_after != Some(receipt.worldline_tick_after) => { @@ -3701,6 +4049,17 @@ pub enum EchoOperationCommitErrorV1 { /// The worldline-local transaction coordinate cannot advance. #[error("executable-operation transaction coordinate overflow")] TransactionCoordinateOverflow, + /// The candidate count cannot be represented by Tick blocker indices. + #[error("executable-operation Action batch has too many candidates")] + TooManyCandidates, + /// Test-only scheduler fault injected before Action Tick construction. + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + #[error("injected executable-operation Action Tick construction failure")] + InjectedTickConstructionFailure, } pub(crate) struct EchoOperationCommitMaterialV1 { @@ -3710,6 +4069,31 @@ pub(crate) struct EchoOperationCommitMaterialV1 { pub patch: WarpTickPatchV1, } +pub(crate) struct SchedulerEchoOperationCandidateV1 { + pub ingress_id: Hash, + pub scope: NodeKey, + pub rule_id: Hash, + pub preparation: EchoOperationPreparationV1, +} + +pub(crate) struct EchoOperationActionBatchCommitMaterialV1 { + pub snapshot: Snapshot, + pub tick_receipt: TickReceipt, + pub patch: WarpTickPatchV1, + pub outcomes: Vec<(Hash, EchoOperationActionOutcomeV1)>, +} + +enum SchedulerEchoOperationDecisionV1 { + Applied(Box), + Obstructed(EchoOperationObstructionV1), + RejectedFootprintConflict { + installed_operation_id: InstalledEchoOperationIdV1, + invocation_id: EchoOperationInvocationIdV1, + preparation_id: PreparedEchoOperationIdV1, + blocked_by: Vec, + }, +} + #[derive(Clone, Copy, Debug)] struct EchoOperationTerminalMaterialV1 { posture: EchoOperationTerminalPostureV1, @@ -3721,6 +4105,258 @@ struct EchoOperationTerminalMaterialV1 { worldline_tick_after: WorldlineTick, } +pub(crate) fn commit_scheduler_action_batch_to_state_v1( + candidates: Vec, + state: &mut WorldlineState, + commit_global_tick: GlobalTick, + policy_id: u32, +) -> Result { + let state_root_before = state.state_root(); + let tx_raw = state + .current_tick() + .as_u64() + .checked_add(1) + .ok_or(EchoOperationCommitErrorV1::TransactionCoordinateOverflow)?; + let tx = TxId::from_raw(tx_raw); + + let mut entries = Vec::with_capacity(candidates.len()); + let mut blocked_by = Vec::with_capacity(candidates.len()); + let mut decisions = Vec::with_capacity(candidates.len()); + let mut accepted_footprints: Vec<(u32, Footprint)> = Vec::new(); + let mut in_slots = Vec::new(); + let mut out_slots = Vec::new(); + let mut ops = Vec::new(); + let mut ordered_rule_ids = Vec::with_capacity(candidates.len()); + + for (entry_index, candidate) in candidates.into_iter().enumerate() { + let entry_index_u32 = u32::try_from(entry_index) + .map_err(|_| EchoOperationCommitErrorV1::TooManyCandidates)?; + ordered_rule_ids.push(candidate.rule_id); + let scope_hash = crate::scope_hash(&candidate.rule_id, &candidate.scope); + match candidate.preparation { + EchoOperationPreparationV1::Obstructed(obstruction) => { + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ), + }); + blocked_by.push(Vec::new()); + decisions.push(( + candidate.ingress_id, + SchedulerEchoOperationDecisionV1::Obstructed(obstruction), + )); + } + EchoOperationPreparationV1::Prepared(prepared) => { + let blockers = accepted_footprints + .iter() + .filter_map(|(accepted_index, footprint)| { + crate::engine_impl::footprints_conflict( + prepared.actual_footprint(), + footprint, + ) + .then_some(*accepted_index) + }) + .collect::>(); + if blockers.is_empty() { + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Applied, + }); + blocked_by.push(Vec::new()); + accepted_footprints + .push((entry_index_u32, prepared.actual_footprint().clone())); + in_slots.extend_from_slice(prepared.patch().in_slots()); + out_slots.extend_from_slice(prepared.patch().out_slots()); + ops.extend_from_slice(prepared.patch().ops()); + decisions.push(( + candidate.ingress_id, + SchedulerEchoOperationDecisionV1::Applied(prepared), + )); + } else { + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Rejected( + TickReceiptRejection::FootprintConflict, + ), + }); + blocked_by.push(blockers.clone()); + decisions.push(( + candidate.ingress_id, + SchedulerEchoOperationDecisionV1::RejectedFootprintConflict { + installed_operation_id: prepared.installed_operation_id(), + invocation_id: prepared.invocation_id(), + preparation_id: prepared.preparation_id(), + blocked_by: blockers, + }, + )); + } + } + } + } + + let rule_pack_id = action_batch_rule_pack_id(&ordered_rule_ids); + let patch = WarpTickPatchV1::new( + policy_id, + rule_pack_id, + TickCommitStatus::Committed, + in_slots, + out_slots, + ops, + ); + let tick_receipt = TickReceipt::new(tx, entries, blocked_by); + let plan_digest = action_batch_plan_digest(tick_receipt.entries()); + let applied = decisions + .iter() + .filter_map(|(_, decision)| match decision { + SchedulerEchoOperationDecisionV1::Applied(prepared) => Some(prepared.as_ref()), + SchedulerEchoOperationDecisionV1::Obstructed(_) + | SchedulerEchoOperationDecisionV1::RejectedFootprintConflict { .. } => None, + }) + .collect::>(); + let rewrites_digest = action_batch_rewrites_digest(&applied); + let composition_digest = action_batch_composition_digest(&applied, patch.digest()); + + let mut next_state = state.warp_state.clone(); + patch.apply_to_state(&mut next_state)?; + let state_root_after = + crate::snapshot::compute_state_root_for_warp_state(&next_state, state.root()); + let parents = state + .last_snapshot + .as_ref() + .map(|snapshot| vec![snapshot.hash]) + .unwrap_or_default(); + let commit_id = compute_commit_hash_v2( + &state_root_after, + &parents, + &patch.digest(), + patch.policy_id(), + ); + let snapshot = Snapshot { + root: *state.root(), + hash: commit_id, + state_root: state_root_after, + parents, + plan_digest, + decision_digest: tick_receipt.digest(), + rewrites_digest, + patch_digest: patch.digest(), + policy_id: patch.policy_id(), + tx, + }; + + let outcomes = decisions + .into_iter() + .map(|(ingress_id, decision)| { + let outcome = match decision { + SchedulerEchoOperationDecisionV1::Applied(prepared) => { + let mut receipt = build_receipt( + &prepared, + EchoOperationTerminalMaterialV1 { + posture: EchoOperationTerminalPostureV1::Committed, + state_root_before, + state_root_after, + commit_id, + tick_receipt_digest: tick_receipt.digest(), + commit_global_tick: Some(commit_global_tick), + worldline_tick_after: WorldlineTick::from_raw(tx_raw), + }, + ); + receipt.committed_patch_digest = Some(patch.digest()); + receipt.composition_digest = Some(composition_digest); + receipt.terminal_outcome_digest = terminal_outcome_digest(&receipt); + receipt.receipt_digest = receipt_digest(&receipt); + EchoOperationActionOutcomeV1::Committed(Box::new(receipt)) + } + SchedulerEchoOperationDecisionV1::Obstructed(obstruction) => { + EchoOperationActionOutcomeV1::Obstructed(obstruction) + } + SchedulerEchoOperationDecisionV1::RejectedFootprintConflict { + installed_operation_id, + invocation_id, + preparation_id, + blocked_by, + } => EchoOperationActionOutcomeV1::RejectedFootprintConflict { + installed_operation_id, + invocation_id, + preparation_id, + blocked_by, + }, + }; + (ingress_id, outcome) + }) + .collect(); + + state.warp_state = next_state; + state.last_snapshot = Some(snapshot.clone()); + state + .tick_history + .push((snapshot.clone(), tick_receipt.clone(), patch.clone())); + state.tx_counter = tx_raw; + + Ok(EchoOperationActionBatchCommitMaterialV1 { + snapshot, + tick_receipt, + patch, + outcomes, + }) +} + +fn action_batch_rule_pack_id(rule_ids: &[Hash]) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_RULE_PACK_DOMAIN); + hasher.update(&(rule_ids.len() as u64).to_le_bytes()); + for rule_id in rule_ids { + hasher.update(rule_id); + } + hasher.finalize().into() +} + +fn action_batch_plan_digest(entries: &[TickReceiptEntry]) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_PLAN_DOMAIN); + hasher.update(&(entries.len() as u64).to_le_bytes()); + for entry in entries { + hasher.update(&entry.rule_id); + hasher.update(&entry.scope_hash); + } + hasher.finalize().into() +} + +fn action_batch_rewrites_digest(prepared: &[&PreparedEchoOperationV1]) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_REWRITES_DOMAIN); + hasher.update(&(prepared.len() as u64).to_le_bytes()); + for member in prepared { + hasher.update(&member.preparation_id().as_hash()); + } + hasher.finalize().into() +} + +fn action_batch_composition_digest( + prepared: &[&PreparedEchoOperationV1], + committed_patch_digest: Hash, +) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_COMPOSITION_DOMAIN); + hasher.update(&committed_patch_digest); + hasher.update(&(prepared.len() as u64).to_le_bytes()); + for member in prepared { + hasher.update(&member.preparation_id().as_hash()); + hasher.update(&member.patch().digest()); + hasher.update(&member.result_id().as_hash()); + hasher.update(&member.evaluation_basis().identity().as_hash()); + hasher.update(&member.actual_footprint_digest()); + } + hasher.finalize().into() +} + pub(crate) fn not_committed_basis_changed( prepared: &PreparedEchoOperationV1, current_state_root: Hash, diff --git a/crates/warp-core/src/engine_impl.rs b/crates/warp-core/src/engine_impl.rs index aea41993..fa382e88 100644 --- a/crates/warp-core/src/engine_impl.rs +++ b/crates/warp-core/src/engine_impl.rs @@ -2988,7 +2988,10 @@ fn attach_footprint_guards( Ok(()) } -fn footprints_conflict(a: &crate::footprint::Footprint, b: &crate::footprint::Footprint) -> bool { +pub(crate) fn footprints_conflict( + a: &crate::footprint::Footprint, + b: &crate::footprint::Footprint, +) -> bool { // IMPORTANT: do not use `Footprint::independent` here yet. // // This logic MUST remain consistent with the scheduler’s footprint conflict diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index b892b0c0..158dc898 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -791,6 +791,51 @@ impl HeadInbox { } } + /// Admits one deterministic execution category without mixing it with + /// other pending categories. + /// + /// The lowest canonical ingress id chooses whether this batch contains the + /// supplied `partition_kind` or everything else. Entries from the other + /// category remain pending for a later Tick. Existing per-Tick limits still + /// bound the selected category. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn admit_partitioned(&mut self, partition_kind: IntentKind) -> Vec { + let Some(first) = self.pending.first_key_value().map(|(_, envelope)| envelope) else { + return Vec::new(); + }; + let selected_partition = matches!( + first.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ); + let limit = match self.policy { + InboxPolicy::Budgeted { max_per_tick } => max_per_tick as usize, + InboxPolicy::AcceptAll | InboxPolicy::KindFilter(_) => usize::MAX, + }; + if limit == 0 { + return Vec::new(); + } + + let mut selected_ids = Vec::new(); + for (ingress_id, envelope) in &self.pending { + let in_partition = matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ); + if in_partition == selected_partition { + selected_ids.push(*ingress_id); + if selected_ids.len() == limit { + break; + } + } + } + selected_ids + .into_iter() + .filter_map(|ingress_id| self.pending.remove(&ingress_id)) + .collect() + } + /// Returns `true` if calling [`HeadInbox::admit`] would yield at least one envelope. #[must_use] pub fn can_admit(&self) -> bool { @@ -899,6 +944,60 @@ mod tests { } } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn partitioned_admission_never_mixes_execution_categories() { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::AcceptAll, + ); + let partition_kind = test_kind(); + let envelopes = [ + make_envelope(partition_kind, b"partition-a"), + make_envelope(other_kind(), b"legacy-a"), + make_envelope(partition_kind, b"partition-b"), + make_envelope(other_kind(), b"legacy-b"), + ]; + let first_is_partition = envelopes + .iter() + .min_by_key(|envelope| envelope.ingress_id()) + .is_some_and(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ) + }); + for envelope in envelopes { + assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); + } + + let first_batch = inbox.admit_partitioned(partition_kind); + assert_eq!(first_batch.len(), 2); + assert!(first_batch.iter().all(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if (*intent_kind == partition_kind) == first_is_partition + ) + })); + assert_eq!(inbox.pending_count(), 2); + + let second_batch = inbox.admit_partitioned(partition_kind); + assert_eq!(second_batch.len(), 2); + assert!(second_batch.iter().all(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if (*intent_kind == partition_kind) != first_is_partition + ) + })); + assert!(inbox.is_empty()); + } + #[test] fn re_ingesting_same_envelope_is_idempotent() { let mut inbox = HeadInbox::new( diff --git a/crates/warp-core/src/lib.rs b/crates/warp-core/src/lib.rs index 02a0ab7b..45ce50ea 100644 --- a/crates/warp-core/src/lib.rs +++ b/crates/warp-core/src/lib.rs @@ -234,17 +234,19 @@ pub use dynamic_binding::{ StructuredBindingRuntime, StructuredRuntimeBindings, }; pub use echo_operation::{ + echo_operation_action_envelope_v1, echo_operation_action_intent_kind_v1, echo_operation_anchored_node_absent_application_basis_v1, echo_operation_anchored_node_application_basis_v1, echo_operation_anchored_node_creation_application_basis_v1, echo_operation_atom_value_digest_v1, echo_operation_create_if_absent_target_profile_identity_v1, echo_operation_package_id_v1, echo_operation_target_profile_identity_v1, AdmittedEchoOperationInvocationV1, - AdmittedExecutableOperationPackageV1, EchoOperationAdmissionErrorKindV1, - EchoOperationAdmissionErrorV1, EchoOperationAdmissionPolicyV1, - EchoOperationAnchoredNodeOccupancyV1, EchoOperationApplicationBasisV1, - EchoOperationArtifactErrorKindV1, EchoOperationArtifactErrorV1, EchoOperationBudgetV1, - EchoOperationCommitErrorV1, EchoOperationEvaluationBasisIdV1, EchoOperationEvaluationBasisV1, + AdmittedExecutableOperationPackageV1, EchoOperationActionOutcomeV1, + EchoOperationAdmissionErrorKindV1, EchoOperationAdmissionErrorV1, + EchoOperationAdmissionPolicyV1, EchoOperationAnchoredNodeOccupancyV1, + EchoOperationApplicationBasisV1, EchoOperationArtifactErrorKindV1, + EchoOperationArtifactErrorV1, EchoOperationBudgetV1, EchoOperationCommitErrorV1, + EchoOperationEvaluationBasisIdV1, EchoOperationEvaluationBasisV1, EchoOperationExecutionEvidenceV1, EchoOperationFootprintContractV1, EchoOperationInstallationErrorKindV1, EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorKindV1, EchoOperationInvocationAdmissionErrorV1, diff --git a/crates/warp-core/src/provenance_codec.rs b/crates/warp-core/src/provenance_codec.rs index a14462a7..7db3aef5 100644 --- a/crates/warp-core/src/provenance_codec.rs +++ b/crates/warp-core/src/provenance_codec.rs @@ -286,6 +286,9 @@ fn push_tick_receipt(out: &mut Vec, receipt: &TickReceipt) { out.push(match entry.disposition { TickReceiptDisposition::Applied => 1, TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) => 2, + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) => 3, }); let blockers = receipt.blocked_by(index); push_len(out, blockers.len()); @@ -611,6 +614,9 @@ impl<'a> RetainedProvenanceCursor<'a> { let disposition = match self.read_u8()? { 1 => TickReceiptDisposition::Applied, 2 => TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict), + 3 => TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ), tag => return Err(unknown_tag("tick receipt disposition", tag)), }; let blocker_count = self.read_count(4)?; diff --git a/crates/warp-core/src/receipt.rs b/crates/warp-core/src/receipt.rs index c0a5ef92..030afade 100644 --- a/crates/warp-core/src/receipt.rs +++ b/crates/warp-core/src/receipt.rs @@ -87,6 +87,14 @@ pub enum TickReceiptPartsError { /// Rejected candidate missing its blocking attribution. entry: usize, }, + /// A non-conflict obstruction cannot attribute an applied blocker. + #[error("obstructed tick receipt entry {entry} carries {blocker_count} blockers")] + ObstructedEntryHasBlockers { + /// Obstructed candidate carrying invalid blocker attribution. + entry: usize, + /// Number of invalid blockers. + blocker_count: usize, + }, } impl TickReceipt { @@ -141,8 +149,19 @@ impl TickReceipt { entry: entry_index, }); } + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) if !blockers.is_empty() => { + return Err(TickReceiptPartsError::ObstructedEntryHasBlockers { + entry: entry_index, + blocker_count: blockers.len(), + }); + } TickReceiptDisposition::Applied - | TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) => {} + | TickReceiptDisposition::Rejected( + TickReceiptRejection::FootprintConflict + | TickReceiptRejection::ExecutableOperationObstruction, + ) => {} } if let Some(pair) = blockers.windows(2).find(|pair| pair[0] >= pair[1]) { return Err(TickReceiptPartsError::BlockersNotStrictlyIncreasing { @@ -285,6 +304,9 @@ impl TickReceiptDisposition { pub enum TickReceiptRejection { /// Candidate footprint conflicts with an already-accepted footprint. FootprintConflict, + /// Echo's bounded executable-operation evaluator returned a typed + /// obstruction and no candidate mutation. + ExecutableOperationObstruction, } fn compute_tick_receipt_digest(entries: &[TickReceiptEntry]) -> Hash { @@ -304,6 +326,9 @@ fn compute_tick_receipt_digest(entries: &[TickReceiptEntry]) -> Hash { let code = match entry.disposition { TickReceiptDisposition::Applied => 1u8, TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) => 2u8, + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) => 3u8, }; hasher.update(&[code]); } diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 87d16c4e..364e8390 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -21,7 +21,8 @@ use crate::{ affected_frontiers_root, build_causal_anchor_admission_transaction, build_executable_operation_installation_transaction, build_executable_operation_tick_transaction, build_recovery_certificate, - build_replayable_tick_transaction, build_submission_acceptance_with_material_transaction, + build_replayable_tick_batch_transaction, build_replayable_tick_transaction, + build_submission_acceptance_with_material_transaction, causal_anchor_frontier_digest_from_evidence, causal_anchor_genesis_frontier_digest, causal_history_genesis_frontier_digest, logical_causal_history_frontier_digest, recover_filesystem_store, recover_from_frames_and_commits, recover_receipt_index, @@ -41,11 +42,12 @@ use crate::{ contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ admit_invocation_v1, admit_package_v1, commit_prepared_to_state, - decode_invocation_route_v1, genesis_commit_id, install_recovered_v1, - installed_from_admitted, not_committed_basis_changed, - not_committed_evaluation_authority_mismatch, not_committed_installation_unavailable, - operation_descent_stack, prepare_operation_v1, recover_committed_execution_receipt_v1, - recover_installation_v1, retain_committed_execution_v1, retain_installation_v1, + decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, + inspect_action_invocation_v1, install_recovered_v1, installed_from_admitted, + not_committed_basis_changed, not_committed_evaluation_authority_mismatch, + not_committed_installation_unavailable, operation_descent_stack, prepare_operation_v1, + recover_action_outcome_v1, recover_committed_execution_receipt_v1, recover_installation_v1, + retain_action_outcome_v1, retain_committed_execution_v1, retain_installation_v1, validate_receipt_installation_v1, EchoOperationEvaluationAuthorityV1, }, provider_contract::admit_provider_contract_package_v1, @@ -54,25 +56,26 @@ use crate::{ CausalAnchorError, CausalAnchorId, CausalAnchorRootSupportPolicy, CausalAnchorSupportError, CausalFrontierRef, ContractInverseAdmissionRequest, ContractInverseContext, ContractInverseDerivation, ContractInverseHistoryObstruction, ContractInverseObstruction, - ContractOperationKind, EchoOperationAdmissionErrorV1, EchoOperationAdmissionPolicyV1, - EchoOperationApplicationBasisV1, EchoOperationArtifactErrorV1, EchoOperationCommitErrorV1, - EchoOperationEvaluationBasisV1, EchoOperationExecutionEvidenceV1, + ContractOperationKind, EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorV1, + EchoOperationAdmissionPolicyV1, EchoOperationApplicationBasisV1, EchoOperationArtifactErrorV1, + EchoOperationCommitErrorV1, EchoOperationEvaluationBasisV1, EchoOperationExecutionEvidenceV1, EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorV1, EchoOperationInvocationAdmissionPolicyV1, EchoOperationPreparationV1, EchoOperationReceiptV1, - Engine, IngressCausalParent, IngressEnvelope, IngressEnvelopeDecodeError, IngressPayload, - IngressSubmissionGeneration, InstalledContractPackage, InstalledContractPackageError, - InstalledContractPackageRecord, InstalledEchoOperationV1, - InstalledProviderContractPackageRecordV1, IntentOutcome, IntentOutcomeDecision, - IntentOutcomeObservation, IntentSubmissionHandle, IntentSubmissionRecord, ObservationArtifact, - ObservationError, ObservationRequest, ObservationService, OpticAdmissionTicket, - PreparedEchoOperationV1, ProvenanceEntry, ProvenanceService, ProvenanceStore, - ProviderContractAdmissionError, ProviderContractAdmissionPolicyV1, - ProviderContractInstallationError, ProviderContractPackageInstallerV1, - ProviderContractPackageProposalV1, ProviderPackageReferenceV1, - ReceiptCorrelationPersistenceRecord, ReceiptCorrelationRecord, RetainedProvenanceError, - RuntimeError, SchedulerCoordinator, StepRecord, TickReceiptRejection, - TicketedRuntimeIngressAuthority, TicketedRuntimeIngressDisposition, - WitnessedSubmissionPersistenceRecord, WitnessedSubmissionPersistenceSnapshot, WorldlineRuntime, + EchoOperationTerminalPostureV1, Engine, IngressCausalParent, IngressEnvelope, + IngressEnvelopeDecodeError, IngressPayload, IngressSubmissionGeneration, + InstalledContractPackage, InstalledContractPackageError, InstalledContractPackageRecord, + InstalledEchoOperationV1, InstalledProviderContractPackageRecordV1, IntentOutcome, + IntentOutcomeDecision, IntentOutcomeObservation, IntentSubmissionHandle, + IntentSubmissionRecord, ObservationArtifact, ObservationError, ObservationRequest, + ObservationService, OpticAdmissionTicket, PreparedEchoOperationV1, ProvenanceEntry, + ProvenanceService, ProvenanceStore, ProviderContractAdmissionError, + ProviderContractAdmissionPolicyV1, ProviderContractInstallationError, + ProviderContractPackageInstallerV1, ProviderContractPackageProposalV1, + ProviderPackageReferenceV1, ReceiptCorrelationPersistenceRecord, ReceiptCorrelationRecord, + RetainedProvenanceError, RuntimeError, SchedulerCoordinator, StepRecord, + TickReceiptDisposition, TickReceiptRejection, TicketedRuntimeIngressAuthority, + TicketedRuntimeIngressDisposition, WitnessedSubmissionPersistenceRecord, + WitnessedSubmissionPersistenceSnapshot, WorldlineRuntime, }; use crate::{Hash, HistoryError}; @@ -83,6 +86,8 @@ const INSTALLED_CONTRACT_HOST_ADMISSION_DIGEST_DOMAIN: &[u8] = b"echo:trusted-host-installed-contract-admission:v1\0"; const PROVIDER_CONTRACT_HOST_ADMISSION_DIGEST_DOMAIN: &[u8] = b"echo:trusted-host-provider-contract-admission:v1\0"; +const ECHO_OPERATION_ACTION_ADMISSION_DIGEST_DOMAIN: &[u8] = + b"echo:trusted-host-executable-operation-action-admission:v1\0"; /// Error returned by the reference trusted host loop. #[derive(Debug, Error)] @@ -131,6 +136,12 @@ pub enum TrustedRuntimeHostError { /// Exact executable-operation retained material was malformed. #[error("trusted runtime host executable-operation artifact error: {0}")] EchoOperationArtifact(#[from] EchoOperationArtifactErrorV1), + /// Runtime-owned Action admission policy is not installed. + #[error("trusted runtime host executable-operation Action admission policy is unavailable")] + EchoOperationActionAdmissionPolicyUnavailable, + /// Runtime-owned admission refused a durably accepted executable Action. + #[error("trusted runtime host executable-operation Action admission error: {0}")] + EchoOperationActionAdmission(#[from] EchoOperationInvocationAdmissionErrorV1), /// Executable-operation installation conflicted with installed authority. #[error("trusted runtime host executable-operation installation error: {0}")] EchoOperationInstallation(#[from] EchoOperationInstallationErrorV1), @@ -329,6 +340,9 @@ pub enum TrustedRuntimeWalError { /// Number of transactions in the attempted durable batch. transaction_count: usize, }, + /// Per-Action receipt records do not describe one exact scheduler Tick. + #[error("trusted runtime WAL scheduler Tick batch is internally inconsistent")] + SchedulerTickBatchMismatch, } /// Live runtime authority category that a WAL activation cannot safely recover. @@ -433,10 +447,27 @@ pub struct TrustedRuntimeWalRecovery { pub installed_echo_operations: Vec, /// Typed receipts reconstructed from executable-operation tick records. pub echo_operation_receipts: Vec, + /// Typed executable-operation Action outcomes reconstructed from + /// scheduler-owned Tick records, keyed by witnessed submission. + pub echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, causal_history_frontiers: Vec, } impl TrustedRuntimeWalRecovery { + /// Re-runs executable Action/Tick correspondence checks after a targeted + /// test mutates recovered evidence. + #[cfg(any(test, feature = "host_test"))] + pub fn validate_echo_operation_action_outcomes_for_test( + &self, + ) -> Result<(), TrustedRuntimeWalError> { + validate_recovered_echo_operation_action_outcomes( + &self.witnessed_submissions, + &self.receipt_correlations, + &self.provenance_entries, + &self.echo_operation_action_outcomes, + ) + } + /// Recomputes the certificate's canonical index root from recovered evidence. /// /// # Errors @@ -454,6 +485,7 @@ impl TrustedRuntimeWalRecovery { causal_anchor_history: &self.causal_anchor_history, installed_echo_operations: &self.installed_echo_operations, echo_operation_receipts: &self.echo_operation_receipts, + echo_operation_action_outcomes: &self.echo_operation_action_outcomes, }) } @@ -606,6 +638,8 @@ pub struct TrustedRuntimeHost { runtime_wal: Option, causal_anchor_support_policy: Option, echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1, + echo_operation_action_admission_policy: Option, + echo_operation_action_outcomes: BTreeMap, } impl TrustedRuntimeHost { @@ -624,6 +658,8 @@ impl TrustedRuntimeHost { runtime_wal: None, causal_anchor_support_policy: None, echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), + echo_operation_action_admission_policy: None, + echo_operation_action_outcomes: BTreeMap::new(), }) } @@ -641,6 +677,8 @@ impl TrustedRuntimeHost { runtime_wal: None, causal_anchor_support_policy: None, echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), + echo_operation_action_admission_policy: None, + echo_operation_action_outcomes: BTreeMap::new(), } } @@ -668,6 +706,25 @@ impl TrustedRuntimeHost { &self.engine } + /// Installs the runtime-owned policy used to admit durably accepted + /// executable-operation Actions before scheduler selection. + pub fn install_echo_operation_action_admission_policy_v1( + &mut self, + policy: EchoOperationInvocationAdmissionPolicyV1, + ) { + self.echo_operation_action_admission_policy = Some(policy); + } + + /// Returns the typed scheduler-owned outcome for one executable-operation + /// Action submission. + #[must_use] + pub fn echo_operation_action_outcome_v1( + &self, + submission_id: &Hash, + ) -> Option<&EchoOperationActionOutcomeV1> { + self.echo_operation_action_outcomes.get(submission_id) + } + /// Admits exact executable-operation package bytes under independent policy. /// /// A successful result is an opaque admission token. It does not install @@ -726,12 +783,13 @@ impl TrustedRuntimeHost { writer_head: crate::WriterHeadKey, application_basis: EchoOperationApplicationBasisV1, ) -> Result { - current_echo_operation_basis( + crate::coordinator::resolve_echo_operation_evaluation_basis_v1( &self.runtime, &self.provenance, writer_head, application_basis, ) + .map_err(TrustedRuntimeHostError::from) } /// Independently admits a canonical installed-operation invocation. @@ -771,11 +829,14 @@ impl TrustedRuntimeHost { ) } - /// Evaluates one admitted operation without mutating the parent state. + /// Transitional host-only seam for evaluating one admitted operation + /// without mutating the parent state. /// - /// Evaluation returns either one complete prepared patch or one typed - /// obstruction. No partial mutation reaches the parent worldline. + /// Application execution must submit an executable-operation Action and + /// let the scheduler invoke private evaluation while constructing a Tick. + /// This direct seam remains hidden for compatibility and focused tests. #[must_use] + #[doc(hidden)] pub fn prepare_echo_operation_v1( &self, admitted: AdmittedEchoOperationInvocationV1, @@ -806,11 +867,13 @@ impl TrustedRuntimeHost { ) } - /// Commits one privately prepared patch only against its exact evaluation basis. + /// Transitional host-only seam for directly committing one privately + /// prepared consequence against its exact evaluation basis. /// - /// A changed basis produces typed noncommit evidence and never retries, - /// rebases, revalidates, or exposes a partial patch. A successful crossing - /// appends one local provenance entry and advances one worldline tick. + /// Application execution must use scheduler-owned executable-operation + /// Actions. This direct seam remains hidden for compatibility and focused + /// tests while that older lifecycle is retired. + #[doc(hidden)] pub fn commit_prepared_echo_operation_v1( &mut self, prepared: Box, @@ -991,6 +1054,11 @@ impl TrustedRuntimeHost { self.engine .restore_recovered_echo_operation_packages_v1(&recovery.installed_echo_operations)?; + self.echo_operation_action_outcomes = recovery + .echo_operation_action_outcomes + .iter() + .map(|(submission_id, _, outcome)| (*submission_id, outcome.clone())) + .collect(); self.runtime = restored_runtime; self.provenance = restored_provenance; self.runtime_wal = Some(runtime_wal); @@ -1034,6 +1102,18 @@ impl TrustedRuntimeHost { Ok(()) } + /// Injects one scheduler Action Tick construction failure for a targeted + /// rollback test. + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + pub fn inject_echo_operation_action_tick_construction_failure_for_test(&mut self) { + self.runtime + .inject_echo_operation_action_tick_construction_failure_for_test(); + } + /// Returns the app-facing surface. This surface can submit and observe, but /// it cannot tick, stage ticketed ingress, register packages, or recover /// scheduler faults. @@ -1489,24 +1569,80 @@ impl TrustedRuntimeHost { ) } + fn admit_pending_echo_operation_actions_v1( + &mut self, + ) -> Result, TrustedRuntimeHostError> { + let pending = self + .runtime + .witnessed_submissions() + .filter_map(|submission| { + matches!( + self.runtime + .observe_intent_outcome(&submission.submission_id), + IntentOutcomeObservation::Pending { .. } + ) + .then(|| { + self.runtime + .witnessed_submission_envelope(&submission.submission_id) + .and_then(|envelope| { + echo_operation_action_invocation_bytes_v1(envelope) + .map(|bytes| (submission.clone(), envelope.clone(), bytes.to_vec())) + }) + }) + .flatten() + }) + .collect::>(); + if pending.is_empty() { + return Ok(BTreeMap::new()); + } + let policy = self + .echo_operation_action_admission_policy + .ok_or(TrustedRuntimeHostError::EchoOperationActionAdmissionPolicyUnavailable)?; + let mut admitted_actions = BTreeMap::new(); + for (submission, envelope, invocation_bytes) in pending { + let admitted = self.admit_echo_operation_invocation_v1(&policy, &invocation_bytes)?; + let admission_digest = echo_operation_action_admission_digest(&submission, &admitted); + self.runtime.ingest_echo_operation_action_v1( + &TicketedRuntimeIngressAuthority::assume_runtime_owner(), + submission.submission_id, + admission_digest, + envelope, + )?; + admitted_actions.insert(submission.ingress_id, admitted); + } + Ok(admitted_actions) + } + /// Runs one scheduler-owned pass. /// /// # Errors /// /// Returns a runtime error if the scheduler pass fails. pub fn tick_once(&mut self) -> Result, TrustedRuntimeHostError> { + let runtime_before = self.runtime.clone(); + let provenance_before = self.provenance.clone(); + let action_outcomes_before = self.echo_operation_action_outcomes.clone(); + let admitted_actions = match self.admit_pending_echo_operation_actions_v1() { + Ok(admitted) => admitted, + Err(error) => { + self.runtime = runtime_before; + self.provenance = provenance_before; + return Err(error); + } + }; let existing_correlations = self .runtime .receipt_correlations() .map(|correlation| correlation.ticketed_ingress_id) .collect::>(); - let runtime_before = self.runtime.clone(); - let provenance_before = self.provenance.clone(); - let records = SchedulerCoordinator::super_tick( - &mut self.runtime, - &mut self.provenance, - &mut self.engine, - )?; + let (records, action_outcomes) = + SchedulerCoordinator::super_tick_with_echo_operation_actions_v1( + &mut self.runtime, + &mut self.provenance, + &mut self.engine, + &admitted_actions, + &self.echo_operation_evaluation_authority, + )?; let mut tick_wal_records = Vec::new(); if self.runtime_wal.is_some() { let new_correlations = self @@ -1527,6 +1663,7 @@ impl TrustedRuntimeHost { Err(error) => { self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; return Err(error.into()); } }; @@ -1536,31 +1673,70 @@ impl TrustedRuntimeHost { Err(error) => { self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; return Err(error); } }; tick_wal_records.push((correlation, decision, state_delta, state_delta_digest)); } } + let action_outcomes_by_ingress = + action_outcomes.iter().cloned().collect::>(); + let mut tick_wal_groups = BTreeMap::new(); + for (correlation, decision, state_delta, state_delta_digest) in tick_wal_records { + let key = ( + correlation.commit_hash, + correlation.tick_receipt_digest, + state_delta_digest, + ); + tick_wal_groups.entry(key).or_insert_with(Vec::new).push(( + correlation, + decision, + state_delta, + state_delta_digest, + )); + } if let Some(runtime_wal) = self.runtime_wal.as_mut() { - if runtime_wal.uses_filesystem_store() && tick_wal_records.len() > 1 { + if runtime_wal.uses_filesystem_store() && tick_wal_groups.len() > 1 { self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; return Err(TrustedRuntimeWalError::FilesystemAtomicBatchUnsupported { transaction_kind: WalTransactionKind::SchedulerTick, - transaction_count: tick_wal_records.len(), + transaction_count: tick_wal_groups.len(), } .into()); } let runtime_wal_before = runtime_wal.clone(); - for (correlation, decision, state_delta, state_delta_digest) in &tick_wal_records { - if let Err(error) = runtime_wal.record_tick_receipt( - correlation, - *decision, - state_delta, - *state_delta_digest, - ) { - if runtime_wal.recover_filesystem_tick_commit_after_error(correlation) { + for group in tick_wal_groups.values() { + let Some((first_correlation, _, state_delta, state_delta_digest)) = group.first() + else { + continue; + }; + let group_has_action_outcomes = group.iter().any(|(correlation, _, _, _)| { + action_outcomes_by_ingress.contains_key(&correlation.ingress_id) + }); + let result = if group.len() == 1 && !group_has_action_outcomes { + runtime_wal.record_tick_receipt( + first_correlation, + group[0].1, + state_delta, + *state_delta_digest, + ) + } else { + let correlations = group + .iter() + .map(|(correlation, decision, _, _)| (correlation.clone(), *decision)) + .collect::>(); + runtime_wal.record_tick_receipt_batch( + &correlations, + &action_outcomes_by_ingress, + state_delta, + *state_delta_digest, + ) + }; + if let Err(error) = result { + if runtime_wal.recover_filesystem_tick_commit_after_error(first_correlation) { continue; } if !runtime_wal.uses_filesystem_store() { @@ -1568,10 +1744,21 @@ impl TrustedRuntimeHost { } self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; return Err(error.into()); } } } + for (ingress_id, outcome) in action_outcomes { + if let Some(submission) = self + .runtime + .witnessed_submissions() + .find(|submission| submission.ingress_id == ingress_id) + { + self.echo_operation_action_outcomes + .insert(submission.submission_id, outcome); + } + } Ok(records) } @@ -1624,37 +1811,6 @@ impl ProviderContractPackageInstallerV1 for TrustedRuntimeHost { } } -fn current_echo_operation_basis( - runtime: &WorldlineRuntime, - provenance: &ProvenanceService, - writer_head: crate::WriterHeadKey, - application_basis: EchoOperationApplicationBasisV1, -) -> Result { - if runtime.heads().get(&writer_head).is_none() { - return Err(RuntimeError::UnknownHead(writer_head).into()); - } - let frontier = runtime - .worldlines() - .get(&writer_head.worldline_id) - .ok_or(RuntimeError::UnknownWorldline(writer_head.worldline_id))?; - let state_root = frontier.state().state_root(); - let (commit_global_tick, commit_id) = match provenance.tip_ref(writer_head.worldline_id)? { - Some(tip) => { - let entry = provenance.entry(tip.worldline_id, tip.worldline_tick)?; - (Some(entry.commit_global_tick), entry.expected.commit_hash) - } - None => (None, genesis_commit_id(writer_head, state_root)), - }; - Ok(EchoOperationEvaluationBasisV1::new( - writer_head, - frontier.frontier_tick(), - commit_global_tick, - state_root, - commit_id, - application_basis, - )) -} - fn ensure_runtime_authority_is_durable( runtime: &WorldlineRuntime, provenance: &ProvenanceService, @@ -2010,10 +2166,17 @@ impl TrustedRuntimeWal { recover_echo_operation_material(&report, &runtime_state.provenance_entries)?; let provenance_entries = runtime_state.provenance_entries; let receipt_correlations = runtime_state.receipt_correlations; + let echo_operation_action_outcomes = runtime_state.echo_operation_action_outcomes; let missing_runtime_state_deltas = runtime_state.missing_runtime_state_deltas; let installed_echo_operations = operation_material.installations; let echo_operation_receipts = operation_material.receipts; validate_recovered_causal_parent_evidence(&witnessed_submissions, &receipt_correlations)?; + validate_recovered_echo_operation_action_outcomes( + &witnessed_submissions, + &receipt_correlations, + &provenance_entries, + &echo_operation_action_outcomes, + )?; let certificate = runtime_wal_recovery_certificate( &report, &RecoveredRuntimeWalIndexEvidence { @@ -2026,6 +2189,7 @@ impl TrustedRuntimeWal { causal_anchor_history: &causal_anchor_history, installed_echo_operations: &installed_echo_operations, echo_operation_receipts: &echo_operation_receipts, + echo_operation_action_outcomes: &echo_operation_action_outcomes, }, )?; Ok(TrustedRuntimeWalRecovery { @@ -2040,6 +2204,7 @@ impl TrustedRuntimeWal { causal_anchor_history, installed_echo_operations, echo_operation_receipts, + echo_operation_action_outcomes, causal_history_frontiers, }) } @@ -2354,6 +2519,103 @@ impl TrustedRuntimeWal { Ok(commit) } + fn record_tick_receipt_batch( + &mut self, + correlations: &[(ReceiptCorrelationRecord, WalTickDecision)], + action_outcomes_by_ingress: &BTreeMap, + state_delta: &WalRuntimeStateDeltaRecord, + state_delta_digest: Hash, + ) -> Result { + let mut correlations = correlations.to_vec(); + correlations.sort_by_key(|(correlation, _)| correlation.ingress_id); + let Some((first_correlation, _)) = correlations.first() else { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + }; + if correlations + .windows(2) + .any(|pair| pair[0].0.ingress_id == pair[1].0.ingress_id) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let same_tick = correlations.iter().all(|(correlation, _)| { + correlation.head_key == first_correlation.head_key + && correlation.commit_global_tick == first_correlation.commit_global_tick + && correlation.worldline_tick_after == first_correlation.worldline_tick_after + && correlation.tick_receipt_digest == first_correlation.tick_receipt_digest + && correlation.commit_hash == first_correlation.commit_hash + }); + if !same_tick { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let first_has_action_outcome = + action_outcomes_by_ingress.contains_key(&first_correlation.ingress_id); + if correlations.iter().any(|(correlation, _)| { + action_outcomes_by_ingress.contains_key(&correlation.ingress_id) + != first_has_action_outcome + }) { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + + let mut next_receipt_frontier = self.receipt_frontier_digest; + let mut records = Vec::with_capacity(correlations.len()); + for (correlation, decision) in &correlations { + let receipt = TickReceiptRecord { + receipt_ref: correlation.causal_receipt_ref, + decision: *decision, + }; + let wal_correlation = WalReceiptCorrelationRecord { + receipt_ref: correlation.causal_receipt_ref, + causal_parent_receipts: correlation.causal_parent_receipts.clone(), + }; + next_receipt_frontier = + receipt_frontier_digest(next_receipt_frontier, receipt, &wal_correlation); + let action_outcome = action_outcomes_by_ingress + .get(&correlation.ingress_id) + .map(|outcome| { + retain_action_outcome_v1( + correlation.submission_id, + correlation.ingress_id, + outcome, + ) + }) + .transpose()?; + records.push((receipt, wal_correlation, action_outcome)); + } + let next_runtime_frontier = runtime_state_frontier_digest( + self.runtime_state_frontier_digest, + first_correlation, + state_delta_digest, + ); + let transaction = build_replayable_tick_batch_transaction( + self.builder( + WalTransactionKind::SchedulerTick, + WalAppendAuthority::TrustedScheduler, + WalTransactionId::from_hash(tick_batch_transaction_digest( + &correlations, + state_delta_digest, + )), + ), + records, + state_delta.to_payload_bytes()?, + vec![ + AffectedFrontier { + kind: AffectedFrontierKind::ReceiptIndex, + before_digest: self.receipt_frontier_digest, + after_digest: next_receipt_frontier, + }, + AffectedFrontier { + kind: AffectedFrontierKind::RuntimeState, + before_digest: self.runtime_state_frontier_digest, + after_digest: next_runtime_frontier, + }, + ], + )?; + let commit = self.append_transaction(transaction)?; + self.receipt_frontier_digest = next_receipt_frontier; + self.runtime_state_frontier_digest = next_runtime_frontier; + Ok(commit) + } + fn record_executable_operation_installation( &mut self, retained_installation_bytes: &[u8], @@ -2860,13 +3122,20 @@ impl TrustedRuntimeWalCursor { submission_frontier_digest(cursor.submission_frontier_digest, record); } WalTransactionKind::SchedulerTick => { - let (receipt, correlation, state_delta_digest, provenance_entry) = - tick_records_from_transaction(transaction)?; - cursor.receipt_frontier_digest = receipt_frontier_digest( - cursor.receipt_frontier_digest, - receipt, - &correlation, - ); + let (records, state_delta_digest, provenance_entry) = + tick_record_batch_from_transaction(transaction)?; + let Some((_, first_correlation, _)) = records.first() else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + for (receipt, correlation, _) in &records { + cursor.receipt_frontier_digest = receipt_frontier_digest( + cursor.receipt_frontier_digest, + *receipt, + correlation, + ); + } cursor.runtime_state_frontier_digest = match provenance_entry { Some(entry) => runtime_state_frontier_digest_from_fields( cursor.runtime_state_frontier_digest, @@ -2880,7 +3149,7 @@ impl TrustedRuntimeWalCursor { ), None => recovered_legacy_runtime_state_frontier_digest( cursor.runtime_state_frontier_digest, - correlation, + first_correlation.clone(), state_delta_digest, ), }; @@ -3576,6 +3845,7 @@ fn validate_operation_receipt_parent_material( struct RecoveredRuntimeStateMaterial { provenance_entries: Vec, receipt_correlations: Vec, + echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, missing_runtime_state_deltas: Vec, } @@ -3584,6 +3854,7 @@ fn recover_runtime_state_delta_material( ) -> Result { let mut entries_by_coordinate = BTreeMap::new(); let mut correlations_by_submission = BTreeMap::new(); + let mut action_outcomes_by_submission = BTreeMap::new(); let mut submission_by_ticket = BTreeMap::new(); let mut missing = Vec::new(); for transaction in &report.transactions { @@ -3606,80 +3877,92 @@ fn recover_runtime_state_delta_material( if transaction.commit.transaction_kind != WalTransactionKind::SchedulerTick { continue; } - let receipt = tick_receipt_from_transaction(transaction)?; - let wal_correlation = tick_correlation_from_transaction(transaction)?; - if wal_correlation.receipt_ref != receipt.receipt_ref { - return Err(decode_trusted_runtime_wal_payload( - WalDecodeError::InvalidEmbeddedFrame, - )); - } + let (records, _, _) = tick_record_batch_from_transaction(transaction)?; let state_delta_frame = required_unique_transaction_frame( transaction, WalRecordKind::RuntimeStateDeltaRecorded, )?; if state_delta_frame.payload.canonical_bytes.len() == core::mem::size_of::() { - missing.push(receipt.receipt_ref.identity_digest()); + missing.extend( + records + .iter() + .map(|(receipt, _, _)| receipt.receipt_ref.identity_digest()), + ); continue; } let state_delta = WalRuntimeStateDeltaRecord::from_payload_bytes( &state_delta_frame.payload.canonical_bytes, )?; - if state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest { + if records.iter().any(|(receipt, _, _)| { + state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest + }) { return Err(RetainedProvenanceError::Inconsistent("state-delta receipt").into()); } let entry = state_delta.provenance_entry().clone(); let head_key = entry .head_key .ok_or(RetainedProvenanceError::MissingHeadKey)?; - let expected_receipt_ref = crate::CausalTickReceiptRef { - worldline_id: entry.worldline_id, - worldline_tick_after: entry - .worldline_tick - .checked_add(1) - .ok_or(RetainedProvenanceError::Inconsistent("worldline tick"))?, - commit_global_tick: entry.commit_global_tick, - commit_hash: entry.expected.commit_hash, - submission_id: receipt.receipt_ref.submission_id, - ticket_digest: receipt.receipt_ref.ticket_digest, - receipt_content_digest: state_delta.receipt_digest(), - }; - if receipt.receipt_ref != expected_receipt_ref { - return Err(RetainedProvenanceError::Inconsistent("causal receipt ref").into()); - } - let persistence = ReceiptCorrelationPersistenceRecord { - submission_id: receipt.receipt_ref.submission_id, - ticket_digest: receipt.receipt_ref.ticket_digest, - causal_receipt_ref: receipt.receipt_ref, - head_key, - commit_global_tick: entry.commit_global_tick, - worldline_tick_after: receipt.receipt_ref.worldline_tick_after, - tick_receipt_digest: receipt.receipt_ref.receipt_content_digest, - commit_hash: entry.expected.commit_hash, - contract: state_delta.contract().cloned(), - causal_parent_receipts: wal_correlation.causal_parent_receipts, - }; - if correlations_by_submission - .get(&receipt.receipt_ref.submission_id) - .is_some_and(|existing| existing != &persistence) - { - return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { - worldline_id: entry.worldline_id, - worldline_tick: entry.worldline_tick, - }); - } - if submission_by_ticket - .insert( - receipt.receipt_ref.ticket_digest, - receipt.receipt_ref.submission_id, - ) - .is_some_and(|existing| existing != receipt.receipt_ref.submission_id) - { - return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { + for (receipt, wal_correlation, action_outcome) in records { + let expected_receipt_ref = crate::CausalTickReceiptRef { worldline_id: entry.worldline_id, - worldline_tick: entry.worldline_tick, - }); + worldline_tick_after: entry + .worldline_tick + .checked_add(1) + .ok_or(RetainedProvenanceError::Inconsistent("worldline tick"))?, + commit_global_tick: entry.commit_global_tick, + commit_hash: entry.expected.commit_hash, + submission_id: receipt.receipt_ref.submission_id, + ticket_digest: receipt.receipt_ref.ticket_digest, + receipt_content_digest: state_delta.receipt_digest(), + }; + if receipt.receipt_ref != expected_receipt_ref { + return Err(RetainedProvenanceError::Inconsistent("causal receipt ref").into()); + } + let persistence = ReceiptCorrelationPersistenceRecord { + submission_id: receipt.receipt_ref.submission_id, + ticket_digest: receipt.receipt_ref.ticket_digest, + causal_receipt_ref: receipt.receipt_ref, + head_key, + commit_global_tick: entry.commit_global_tick, + worldline_tick_after: receipt.receipt_ref.worldline_tick_after, + tick_receipt_digest: receipt.receipt_ref.receipt_content_digest, + commit_hash: entry.expected.commit_hash, + contract: state_delta.contract().cloned(), + causal_parent_receipts: wal_correlation.causal_parent_receipts, + }; + if correlations_by_submission + .get(&receipt.receipt_ref.submission_id) + .is_some_and(|existing| existing != &persistence) + { + return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { + worldline_id: entry.worldline_id, + worldline_tick: entry.worldline_tick, + }); + } + if submission_by_ticket + .insert( + receipt.receipt_ref.ticket_digest, + receipt.receipt_ref.submission_id, + ) + .is_some_and(|existing| existing != receipt.receipt_ref.submission_id) + { + return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { + worldline_id: entry.worldline_id, + worldline_tick: entry.worldline_tick, + }); + } + correlations_by_submission.insert(receipt.receipt_ref.submission_id, persistence); + if let Some((submission_id, ingress_id, outcome)) = action_outcome { + if submission_id != receipt.receipt_ref.submission_id + || action_outcomes_by_submission + .get(&submission_id) + .is_some_and(|existing| existing != &(ingress_id, outcome.clone())) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + action_outcomes_by_submission.insert(submission_id, (ingress_id, outcome)); + } } - correlations_by_submission.insert(receipt.receipt_ref.submission_id, persistence); let coordinate = (entry.worldline_id, entry.worldline_tick); if entries_by_coordinate .get(&coordinate) @@ -3711,9 +3994,14 @@ fn recover_runtime_state_delta_material( }); missing.sort_unstable(); missing.dedup(); + let echo_operation_action_outcomes = action_outcomes_by_submission + .into_iter() + .map(|(submission_id, (ingress_id, outcome))| (submission_id, ingress_id, outcome)) + .collect(); Ok(RecoveredRuntimeStateMaterial { provenance_entries: entries, receipt_correlations: correlations, + echo_operation_action_outcomes, missing_runtime_state_deltas: missing, }) } @@ -3743,6 +4031,162 @@ fn validate_recovered_causal_parent_evidence( Ok(()) } +fn validate_recovered_echo_operation_action_outcomes( + witnessed_submissions: &WitnessedSubmissionPersistenceSnapshot, + receipt_correlations: &[ReceiptCorrelationPersistenceRecord], + provenance_entries: &[ProvenanceEntry], + outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], +) -> Result<(), TrustedRuntimeWalError> { + let envelopes = witnessed_submissions + .records() + .iter() + .map(|record| (record.submission.submission_id, &record.envelope)) + .collect::>(); + let correlations = receipt_correlations + .iter() + .map(|correlation| (correlation.submission_id, correlation)) + .collect::>(); + let provenance = provenance_entries + .iter() + .map(|entry| (entry.expected.commit_hash, entry)) + .collect::>(); + let mut outcomes_by_commit = BTreeMap::< + Hash, + Vec<( + Hash, + &EchoOperationActionOutcomeV1, + &ReceiptCorrelationPersistenceRecord, + crate::echo_operation::EchoOperationActionInvocationEvidenceV1, + )>, + >::new(); + for (submission_id, ingress_id, outcome) in outcomes { + let envelope = envelopes + .get(submission_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let correlation = correlations + .get(submission_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let invocation_bytes = echo_operation_action_invocation_bytes_v1(envelope) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let invocation = inspect_action_invocation_v1(invocation_bytes) + .map_err(|_| TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + if envelope.ingress_id() != *ingress_id { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + if receipt.tick_receipt_digest() != correlation.tick_receipt_digest + || receipt.commit_id() != correlation.commit_hash + || receipt.commit_global_tick() != Some(correlation.commit_global_tick) + || receipt.worldline_tick_after() != correlation.worldline_tick_after + || receipt.evaluation_basis().writer_head() != correlation.head_key + || receipt.package_id() != invocation.package_id + || receipt.invocation_id() != invocation.invocation_id + || receipt.invocation_bytes_digest() != invocation.invocation_bytes_digest + || receipt.evaluation_basis() != invocation.evaluation_basis + || receipt.terminal_posture() != EchoOperationTerminalPostureV1::Committed + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + if obstruction.package_id() != invocation.package_id + || obstruction.invocation_id() != invocation.invocation_id + || obstruction.evaluation_basis_id() != invocation.evaluation_basis.identity() + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict { + invocation_id, + blocked_by, + .. + } => { + if blocked_by.is_empty() + || blocked_by.windows(2).any(|pair| pair[0] >= pair[1]) + || *invocation_id != invocation.invocation_id + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + } + outcomes_by_commit + .entry(correlation.commit_hash) + .or_default() + .push((*ingress_id, outcome, correlation, invocation)); + } + for (commit_hash, mut group) in outcomes_by_commit { + let entry = provenance + .get(&commit_hash) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let tick_receipt = entry + .tick_receipt + .as_ref() + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + group.sort_by_key(|(ingress_id, _, _, _)| *ingress_id); + if group.len() != tick_receipt.entries().len() { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + for (index, (_, outcome, correlation, invocation)) in group.into_iter().enumerate() { + if entry.commit_global_tick != correlation.commit_global_tick + || entry.head_key != Some(correlation.head_key) + || entry.worldline_id != correlation.head_key.worldline_id + || tick_receipt.digest() != correlation.tick_receipt_digest + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let tick_entry = tick_receipt + .entries() + .get(index) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let blockers = tick_receipt.blocked_by(index); + if tick_entry.scope != invocation.scope + || tick_entry.scope_hash + != crate::scope_hash(&tick_entry.rule_id, &tick_entry.scope) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + if tick_entry.disposition != TickReceiptDisposition::Applied + || !blockers.is_empty() + || tick_entry.rule_id != receipt.installed_operation_id().as_hash() + || receipt.state_root_after() != entry.expected.state_root + || receipt.committed_patch_digest() != Some(entry.expected.patch_digest) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + if tick_entry.disposition + != TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) + || !blockers.is_empty() + || tick_entry.rule_id != obstruction.installed_operation_id().as_hash() + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict { + installed_operation_id, + blocked_by, + .. + } => { + if tick_entry.disposition + != TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) + || tick_entry.rule_id != installed_operation_id.as_hash() + || blockers != blocked_by + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + } + } + } + Ok(()) +} + fn operation_installation_from_transaction( transaction: &crate::causal_wal::WalRecoveredTransaction, ) -> Result<(InstalledEchoOperationV1, Vec), TrustedRuntimeWalError> { @@ -3789,6 +4233,7 @@ fn operation_tick_records_from_transaction( Ok((receipt, state_delta, state_delta_digest)) } +#[cfg(test)] fn tick_records_from_transaction( transaction: &crate::causal_wal::WalRecoveredTransaction, ) -> Result< @@ -3800,15 +4245,119 @@ fn tick_records_from_transaction( ), TrustedRuntimeWalError, > { - let receipt = tick_receipt_from_transaction(transaction)?; - let correlation = tick_correlation_from_transaction(transaction)?; - if correlation.receipt_ref != receipt.receipt_ref { + let (mut records, state_delta_digest, provenance_entry) = + tick_record_batch_from_transaction(transaction)?; + if records.len() != 1 { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let (receipt, correlation, action_outcome) = records.remove(0); + if action_outcome.is_some() { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + Ok((receipt, correlation, state_delta_digest, provenance_entry)) +} + +type RecoveredSchedulerTickMember = ( + TickReceiptRecord, + WalReceiptCorrelationRecord, + Option<(Hash, Hash, EchoOperationActionOutcomeV1)>, +); +type RecoveredSchedulerTickBatch = ( + Vec, + Hash, + Option, +); + +fn tick_record_batch_from_transaction( + transaction: &crate::causal_wal::WalRecoveredTransaction, +) -> Result { + let state_delta_index = transaction + .frames + .iter() + .position(|frame| frame.header.record_kind == WalRecordKind::RuntimeStateDeltaRecorded) + .ok_or_else(missing_trusted_runtime_record)?; + if state_delta_index == 0 || state_delta_index + 1 != transaction.frames.len() { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let mut records = Vec::new(); + let mut index = 0; + while index < state_delta_index { + let Some(receipt_frame) = transaction.frames.get(index) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + let Some(correlation_frame) = transaction.frames.get(index + 1) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + if receipt_frame.header.record_kind != WalRecordKind::TickReceiptRecorded + || correlation_frame.header.record_kind != WalRecordKind::ReceiptCorrelationRecorded + { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let receipt = TickReceiptRecord::from_payload_bytes(&receipt_frame.payload.canonical_bytes) + .map_err(decode_trusted_runtime_wal_payload)?; + let correlation = WalReceiptCorrelationRecord::from_payload_bytes( + &correlation_frame.payload.canonical_bytes, + ) + .map_err(decode_trusted_runtime_wal_payload)?; + if correlation.receipt_ref != receipt.receipt_ref { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + index += 2; + let action_outcome = if transaction.frames.get(index).is_some_and(|frame| { + frame.header.record_kind == WalRecordKind::ExecutableOperationActionOutcomeRecorded + }) { + let frame = &transaction.frames[index]; + index += 1; + let recovered = recover_action_outcome_v1(&frame.payload.canonical_bytes)?; + if recovered.0 != receipt.receipt_ref.submission_id { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + Some(recovered) + } else { + None + }; + records.push((receipt, correlation, action_outcome)); + } + let action_outcome_count = records + .iter() + .filter(|(_, _, action_outcome)| action_outcome.is_some()) + .count(); + if action_outcome_count != 0 && action_outcome_count != records.len() { return Err(decode_trusted_runtime_wal_payload( WalDecodeError::InvalidEmbeddedFrame, )); } - let state_delta_frame = - required_unique_transaction_frame(transaction, WalRecordKind::RuntimeStateDeltaRecorded)?; + if action_outcome_count != 0 { + let mut previous_ingress_id = None; + for (_, _, action_outcome) in &records { + let (_, ingress_id, _) = action_outcome.as_ref().ok_or_else(|| { + decode_trusted_runtime_wal_payload(WalDecodeError::InvalidEmbeddedFrame) + })?; + if previous_ingress_id.is_some_and(|previous| previous >= *ingress_id) { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + previous_ingress_id = Some(*ingress_id); + } + } + let state_delta_frame = &transaction.frames[state_delta_index]; let (state_delta_digest, provenance_entry) = if state_delta_frame.payload.canonical_bytes.len() == core::mem::size_of::() { @@ -3825,7 +4374,9 @@ fn tick_records_from_transaction( let state_delta = WalRuntimeStateDeltaRecord::from_payload_bytes( &state_delta_frame.payload.canonical_bytes, )?; - if state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest { + if records.iter().any(|(receipt, _, _)| { + state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest + }) { return Err(RetainedProvenanceError::Inconsistent("state-delta receipt").into()); } ( @@ -3833,25 +4384,7 @@ fn tick_records_from_transaction( Some(state_delta.provenance_entry().clone()), ) }; - Ok((receipt, correlation, state_delta_digest, provenance_entry)) -} - -fn tick_receipt_from_transaction( - transaction: &crate::causal_wal::WalRecoveredTransaction, -) -> Result { - let receipt_frame = - required_unique_transaction_frame(transaction, WalRecordKind::TickReceiptRecorded)?; - TickReceiptRecord::from_payload_bytes(&receipt_frame.payload.canonical_bytes) - .map_err(decode_trusted_runtime_wal_payload) -} - -fn tick_correlation_from_transaction( - transaction: &crate::causal_wal::WalRecoveredTransaction, -) -> Result { - let correlation_frame = - required_unique_transaction_frame(transaction, WalRecordKind::ReceiptCorrelationRecorded)?; - WalReceiptCorrelationRecord::from_payload_bytes(&correlation_frame.payload.canonical_bytes) - .map_err(decode_trusted_runtime_wal_payload) + Ok((records, state_delta_digest, provenance_entry)) } fn required_unique_transaction_frame( @@ -4374,6 +4907,22 @@ fn provider_contract_host_admission_digest( hasher.finalize().into() } +fn echo_operation_action_admission_digest( + submission: &IntentSubmissionRecord, + admitted: &AdmittedEchoOperationInvocationV1, +) -> Hash { + let mut hasher = blake3::Hasher::new(); + hasher.update(ECHO_OPERATION_ACTION_ADMISSION_DIGEST_DOMAIN); + hasher.update(&submission.submission_id); + hasher.update(&submission.ingress_id); + hasher.update(submission.head_key.worldline_id.as_bytes()); + hasher.update(submission.head_key.head_id.as_bytes()); + hasher.update(&admitted.package_id().as_hash()); + hasher.update(&admitted.installed_operation_id().as_hash()); + hasher.update(&admitted.admission_id().as_hash()); + hasher.finalize().into() +} + fn submission_frontier_digest(previous: Hash, record: SubmissionAcceptanceRecord) -> Hash { let mut hasher = blake3::Hasher::new(); hasher.update(TRUSTED_RUNTIME_WAL_DOMAIN); @@ -4414,6 +4963,10 @@ fn wal_tick_decision_from_observation( reason: TickReceiptRejection::FootprintConflict, .. } => WalTickDecision::RejectedFootprintConflict, + IntentOutcomeDecision::Rejected { + reason: TickReceiptRejection::ExecutableOperationObstruction, + .. + } => WalTickDecision::Obstructed, IntentOutcomeDecision::NoMatchingReceiptEntry { .. } => { return Err(TrustedRuntimeWalError::TickOutcomeUnavailable { submission_id: correlation.submission_id, @@ -5653,6 +6206,25 @@ fn tick_transaction_digest( hasher.finalize().into() } +fn tick_batch_transaction_digest( + correlations: &[(ReceiptCorrelationRecord, WalTickDecision)], + state_delta_digest: Hash, +) -> Hash { + let mut hasher = blake3::Hasher::new(); + hasher.update(TRUSTED_RUNTIME_WAL_DOMAIN); + hasher.update(b"tick-batch-transaction:v1\0"); + hasher.update(&(correlations.len() as u64).to_le_bytes()); + for (correlation, decision) in correlations { + hasher.update(&correlation.ticketed_ingress_id); + hasher.update(&correlation.causal_receipt_ref.to_canonical_bytes()); + hasher.update(&correlation.ingress_id); + hash_causal_parent_receipts(&mut hasher, &correlation.causal_parent_receipts); + hasher.update(&[wal_tick_decision_code(*decision)]); + } + hasher.update(&state_delta_digest); + hasher.finalize().into() +} + fn receipt_frontier_digest( previous: Hash, receipt: TickReceiptRecord, @@ -5800,6 +6372,7 @@ struct RecoveredRuntimeWalIndexEvidence<'a> { causal_anchor_history: &'a [WitnessedCausalAnchorAdmission], installed_echo_operations: &'a [InstalledEchoOperationV1], echo_operation_receipts: &'a [EchoOperationReceiptV1], + echo_operation_action_outcomes: &'a [(Hash, Hash, EchoOperationActionOutcomeV1)], } fn runtime_wal_recovery_certificate( @@ -5839,6 +6412,7 @@ fn recovered_runtime_wal_indexes_root( causal_anchor_root, indexes.installed_echo_operations, indexes.echo_operation_receipts, + indexes.echo_operation_action_outcomes, ) } @@ -5846,8 +6420,9 @@ fn recovered_echo_operation_index_root( base_root: Hash, installations: &[InstalledEchoOperationV1], receipts: &[EchoOperationReceiptV1], + action_outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], ) -> Result { - if installations.is_empty() && receipts.is_empty() { + if installations.is_empty() && receipts.is_empty() && action_outcomes.is_empty() { return Ok(base_root); } let mut hasher = blake3::Hasher::new(); @@ -5865,6 +6440,12 @@ fn recovered_echo_operation_index_root( hasher.update(&(bytes.len() as u64).to_le_bytes()); hasher.update(&bytes); } + hasher.update(&(action_outcomes.len() as u64).to_le_bytes()); + for (submission_id, ingress_id, outcome) in action_outcomes { + let bytes = retain_action_outcome_v1(*submission_id, *ingress_id, outcome)?; + hasher.update(&(bytes.len() as u64).to_le_bytes()); + hasher.update(&bytes); + } Ok(hasher.finalize().into()) } diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 42d8de85..82f61737 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -12,22 +12,24 @@ use std::{ use bytes::Bytes; use echo_edict_canonical::{decode_canonical_cbor_v1, encode_canonical_cbor_v1, CanonicalValueV1}; use warp_core::causal_wal::{ - AffectedFrontier, AffectedFrontierKind, Lsn, PayloadCodecId, PayloadSchemaId, - WalAppendAuthority, WalBuildError, WalDurabilityMode, WalRecordKind, WalSegmentId, - WalTransactionBuilder, WalTransactionId, WalTransactionKind, WalValidationError, WriterEpochId, + AffectedFrontier, AffectedFrontierKind, FilesystemWalFaultPlan, FilesystemWalFaultTarget, Lsn, + PayloadCodecId, PayloadSchemaId, WalAppendAuthority, WalBuildError, WalDurabilityMode, + WalRecordKind, WalSegmentId, WalTransactionBuilder, WalTransactionId, WalTransactionKind, + WalValidationError, WriterEpochId, }; use warp_core::{ make_head_id, make_node_id, make_type_id, make_warp_id, AtomPayload, AttachmentValue, - EchoOperationAdmissionErrorKindV1, EchoOperationAdmissionPolicyV1, - EchoOperationAnchoredNodeOccupancyV1, EchoOperationApplicationBasisV1, - EchoOperationArtifactErrorKindV1, EchoOperationBudgetV1, - EchoOperationInvocationAdmissionErrorKindV1, EchoOperationInvocationAdmissionPolicyV1, - EchoOperationInvocationV1, EchoOperationObstructionKindV1, EchoOperationPreparationV1, - EchoOperationProgramV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, - EngineBuilder, ExecutableOperationPackageV1, GraphStore, InboxPolicy, InstalledEchoOperationV1, - NodeKey, NodeRecord, PlaybackMode, RuntimeWalActivationGap, SchedulerKind, TrustedRuntimeHost, - TrustedRuntimeHostError, TrustedRuntimeWalConfig, TrustedRuntimeWalError, WorldlineId, - WorldlineRuntime, WorldlineState, WriterHead, WriterHeadKey, + EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorKindV1, + EchoOperationAdmissionPolicyV1, EchoOperationAnchoredNodeOccupancyV1, + EchoOperationApplicationBasisV1, EchoOperationArtifactErrorKindV1, EchoOperationBudgetV1, + EchoOperationCommitErrorV1, EchoOperationInvocationAdmissionErrorKindV1, + EchoOperationInvocationAdmissionPolicyV1, EchoOperationInvocationV1, + EchoOperationObstructionKindV1, EchoOperationPreparationV1, EchoOperationProgramV1, + EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, EngineBuilder, + ExecutableOperationPackageV1, GraphStore, InboxPolicy, IngressTarget, InstalledEchoOperationV1, + NodeKey, NodeRecord, PlaybackMode, RuntimeError, RuntimeWalActivationGap, SchedulerKind, + TrustedRuntimeHost, TrustedRuntimeHostError, TrustedRuntimeWalConfig, TrustedRuntimeWalError, + WorldlineId, WorldlineRuntime, WorldlineState, WriterHead, WriterHeadKey, }; const OPERATION_COORDINATE: &str = "echo.fixture.SetAnchoredAtom.v1"; @@ -472,6 +474,33 @@ fn canonical_invocation( .expect("fixture invocation encodes canonically") } +fn action_invocation( + host: &TrustedRuntimeHost, + installed: &InstalledEchoOperationV1, + head_key: WriterHeadKey, + node: NodeKey, + current: &[u8], + replacement: &[u8], +) -> Vec { + let attachment_type = make_type_id("operation-fixture-atom"); + let application_basis = warp_core::echo_operation_anchored_node_application_basis_v1( + node, + attachment_type, + current, + ); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis) + .expect("Echo resolves the shared parent and candidate application basis"); + canonical_invocation( + installed, + evaluation_basis, + node, + warp_core::echo_operation_atom_value_digest_v1(attachment_type, current), + replacement.to_vec(), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + ) +} + fn replace_canonical_map_field( bytes: &[u8], field_name: &str, @@ -1523,6 +1552,10 @@ fn operation_wal_codes_append_without_renumbering_legacy_evidence() { WalRecordKind::ExecutableOperationStateDeltaRecorded.stable_code(), 27 ); + assert_eq!( + WalRecordKind::ExecutableOperationActionOutcomeRecorded.stable_code(), + 28 + ); assert_eq!(AffectedFrontierKind::RuntimeState.stable_code(), 2); assert_eq!(AffectedFrontierKind::CausalAnchorIndex.stable_code(), 8); @@ -2541,3 +2574,595 @@ fn update_precondition_refuses_when_the_node_has_the_wrong_type() { EchoOperationObstructionKindV1::NodeTypeMismatch ); } + +#[test] +fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { + let wal_dir = TempWalDir::new(); + let second_node_id = make_node_id("operation-fixture-second"); + let attachment_type = make_type_id("operation-fixture-atom"); + let package_id; + let first_submission_id; + let second_submission_id; + + { + let (mut host, head_key, first_node, second_node) = fixture_host_with_bare_node( + second_node_id, + make_type_id("operation-fixture-node"), + Some((attachment_type, b"second-before")), + ); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the scheduler fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + package_id = installed.package_id(); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let first_invocation = action_invocation( + &host, + &installed, + head_key, + first_node, + b"before", + b"first-after", + ); + let second_invocation = action_invocation( + &host, + &installed, + head_key, + second_node, + b"second-before", + b"second-after", + ); + let first_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + first_invocation, + ) + .expect("the first invocation is one canonical Action envelope"); + let second_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + second_invocation, + ) + .expect("the second invocation is one canonical Action envelope"); + + first_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(first_envelope) + .expect("the first Action is durable before acknowledgement") + .submission_id; + second_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(second_envelope) + .expect("the second Action is durable before acknowledgement") + .submission_id; + + assert_eq!( + host.runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline remains live") + .state() + .current_tick() + .as_u64(), + 0, + "submission and runtime admission cannot evaluate the Action" + ); + + let steps = host + .tick_once() + .expect("the scheduler privately evaluates and commits both Actions"); + assert_eq!(steps.len(), 1); + assert_eq!(steps[0].admitted_count, 2); + assert_eq!(steps[0].worldline_tick_after.as_u64(), 1); + + for submission_id in [first_submission_id, second_submission_id] { + let Some(EchoOperationActionOutcomeV1::Committed(receipt)) = + host.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("each Action has one typed committed outcome"); + }; + assert_eq!(receipt.worldline_tick_after().as_u64(), 1); + } + + let state = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the committed worldline remains available") + .state(); + assert_eq!( + state + .store(&first_node.warp_id) + .and_then(|store| store.node_attachment(&first_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"first-after"), + ))) + ); + assert_eq!( + state + .store(&second_node.warp_id) + .and_then(|store| store.node_attachment(&second_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"second-after"), + ))) + ); + let runtime_wal = host + .runtime_wal() + .expect("the scheduler WAL remains enabled"); + let commits = runtime_wal.commits(); + assert_eq!( + commits + .iter() + .map(|commit| commit.transaction_kind) + .collect::>(), + vec![ + WalTransactionKind::ExecutableOperationInstallation, + WalTransactionKind::SubmissionIntake, + WalTransactionKind::SubmissionIntake, + WalTransactionKind::SchedulerTick, + ], + "one scheduler-owned Tick must be one WAL transaction" + ); + let scheduler_transaction_id = commits + .last() + .expect("the scheduler Tick has one commit marker") + .transaction_id; + assert_eq!( + runtime_wal + .frames() + .into_iter() + .filter(|frame| frame.header.transaction_id == scheduler_transaction_id) + .map(|frame| frame.header.record_kind) + .collect::>(), + vec![ + WalRecordKind::TickReceiptRecorded, + WalRecordKind::ReceiptCorrelationRecorded, + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + WalRecordKind::TickReceiptRecorded, + WalRecordKind::ReceiptCorrelationRecorded, + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + WalRecordKind::RuntimeStateDeltaRecorded, + ], + "one composite Tick retains per-Action decisions and one state delta" + ); + + let mut adversarial = runtime_wal + .recover_read_only() + .expect("the honest composite Tick recovers"); + assert!( + adversarial + .echo_operation_action_outcomes + .windows(2) + .all(|pair| pair[0].1 < pair[1].1), + "Action outcome records are retained in canonical ingress order" + ); + let (first, second) = adversarial.echo_operation_action_outcomes.split_at_mut(1); + std::mem::swap(&mut first[0].2, &mut second[0].2); + assert!(matches!( + adversarial.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + } + + let (mut recovered, head_key, first_node, second_node) = fixture_host_with_bare_node( + second_node_id, + make_type_id("operation-fixture-node"), + Some((attachment_type, b"second-before")), + ); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("a fresh host recovers the composite Action Tick"); + assert!(recovered + .engine() + .installed_echo_operation_package_v1(package_id) + .is_some()); + for submission_id in [first_submission_id, second_submission_id] { + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + } + let state = recovered + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the recovered worldline exists") + .state(); + assert_eq!(state.current_tick().as_u64(), 1); + assert_eq!( + state + .store(&first_node.warp_id) + .and_then(|store| store.node_attachment(&first_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"first-after"), + ))) + ); + assert_eq!( + state + .store(&second_node.warp_id) + .and_then(|store| store.node_attachment(&second_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"second-after"), + ))) + ); +} + +#[test] +fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { + let wal_dir = TempWalDir::new(); + let submission_id; + let package_id; + + { + let (mut host, head_key, node) = fixture_host(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the pending-Action fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + package_id = installed.package_id(); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"after-restart", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes a canonical Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("Action acceptance is durable before acknowledgement") + .submission_id; + assert!(matches!( + host.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + assert_eq!( + host.runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline remains available") + .state() + .current_tick() + .as_u64(), + 0 + ); + } + + let (mut recovered, head_key, node) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the accepted pre-Tick Action recovers"); + assert!(recovered + .engine() + .installed_echo_operation_package_v1(package_id) + .is_some()); + assert!(matches!( + recovered.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + recovered.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let steps = recovered + .tick_once() + .expect("the scheduler admits and evaluates recovered pending work"); + assert_eq!(steps.len(), 1); + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + let state = recovered + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the recovered Tick advances the worldline") + .state(); + assert_eq!(state.current_tick().as_u64(), 1); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"after-restart"), + ))) + ); +} + +#[test] +fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { + let wal_dir = TempWalDir::new(); + let submission_id; + let state_root_before; + + { + let (mut host, head_key, node) = fixture_host(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the obstruction fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + state_root_before = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline exists") + .state() + .state_root(); + let application_basis = application_basis(); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis) + .expect("Echo resolves the honest parent basis"); + let invocation = canonical_invocation( + &installed, + evaluation_basis, + node, + digest("intentionally-wrong-value-precondition"), + b"must-not-appear".to_vec(), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the obstructed invocation is still a canonical Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the Action itself is durably accepted") + .submission_id; + host.tick_once() + .expect("typed obstruction is a lawful scheduler decision"); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("the Action retains its typed evaluator obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::PreconditionMismatch + ); + let state = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the decided worldline exists") + .state(); + assert_eq!(state.state_root(), state_root_before); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); + assert!(state + .tick_history() + .last() + .expect("the obstruction is represented by one decided Tick") + .2 + .ops() + .is_empty()); + } + + let (mut recovered, head_key, node) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("a fresh host recovers the typed obstruction"); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + recovered.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("recovery restores the typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::PreconditionMismatch + ); + let state = recovered + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the recovered worldline exists") + .state(); + assert_eq!(state.state_root(), state_root_before); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); +} + +#[test] +fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { + let wal_dir = TempWalDir::new(); + let submission_id; + let head_key; + let node; + + { + let (mut host, fixture_head, fixture_node) = fixture_host(); + head_key = fixture_head; + node = fixture_node; + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the WAL-failure fixture opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"must-rollback", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes one Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("acceptance commits before the injected Tick failure") + .submission_id; + host.inject_runtime_wal_filesystem_fault_for_test(FilesystemWalFaultPlan::fail_next( + FilesystemWalFaultTarget::AppendFrame, + )) + .expect("the test injects one scheduler-WAL append failure"); + assert!(host.tick_once().is_err()); + assert!(host + .echo_operation_action_outcome_v1(&submission_id) + .is_none()); + assert!(matches!( + host.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + let state = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the rolled-back worldline remains available") + .state(); + assert_eq!(state.current_tick().as_u64(), 0); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); + } + + let (mut recovered, _, _) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("recovery discards the uncommitted Tick tail"); + assert!(matches!( + recovered.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + assert!(recovered + .echo_operation_action_outcome_v1(&submission_id) + .is_none()); + recovered.install_echo_operation_action_admission_policy_v1(invocation_policy()); + recovered + .tick_once() + .expect("the recovered pending Action remains executable"); + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + +#[test] +fn scheduler_tick_construction_failure_publishes_no_action_state_or_receipt() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the construction-failure fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let state_root_before = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline exists") + .state() + .state_root(); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"must-not-commit", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes one Action"); + let submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("acceptance is durable before Tick construction") + .submission_id; + host.inject_echo_operation_action_tick_construction_failure_for_test(); + + let error = host + .tick_once() + .expect_err("the poisoned coordinate prevents Tick construction"); + assert!( + matches!( + &error, + TrustedRuntimeHostError::Runtime(error) + if matches!( + error.as_ref(), + RuntimeError::EchoOperationCommit( + EchoOperationCommitErrorV1::InjectedTickConstructionFailure + ) + ) + ), + "unexpected construction error: {error:?}" + ); + assert!(host + .echo_operation_action_outcome_v1(&submission_id) + .is_none()); + assert!(matches!( + host.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { .. } + )); + let frontier = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the failed Tick leaves the worldline available"); + assert_eq!(frontier.frontier_tick().as_u64(), 0); + assert_eq!(frontier.state().state_root(), state_root_before); + assert!(frontier.state().tick_history().is_empty()); + assert_eq!( + frontier + .state() + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); + assert_eq!( + host.runtime_wal() + .expect("the WAL remains enabled") + .commits() + .iter() + .map(|commit| commit.transaction_kind) + .collect::>(), + vec![ + WalTransactionKind::ExecutableOperationInstallation, + WalTransactionKind::SubmissionIntake, + ] + ); +} From 73679d6cb8c3ca7bc272d06ad27f06561dcd1db6 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 04:57:29 -0700 Subject: [PATCH 02/64] fix: align scheduler Action pre-push features --- scripts/verify-local.sh | 6 ++++-- tests/hooks/test_verify_local.sh | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/verify-local.sh b/scripts/verify-local.sh index 89c6fcf0..f1e080eb 100755 --- a/scripts/verify-local.sh +++ b/scripts/verify-local.sh @@ -1157,10 +1157,12 @@ pre_push_feature_string_for_test_target() { printf '%s\n' "native_rule_bootstrap,host_test" ;; warp-core:external_consumer_contract_fixture_tests|\ - warp-core:provider_contract_admission_tests|\ - warp-core:executable_operation_pipeline_tests) + warp-core:provider_contract_admission_tests) printf '%s\n' "native_rule_bootstrap,trusted_runtime" ;; + warp-core:executable_operation_pipeline_tests) + printf '%s\n' "native_rule_bootstrap,trusted_runtime,host_test" + ;; warp-core:installed_contract_registry_tests) printf '%s\n' "native_rule_bootstrap" ;; diff --git a/tests/hooks/test_verify_local.sh b/tests/hooks/test_verify_local.sh index 517b4135..feccd77c 100755 --- a/tests/hooks/test_verify_local.sh +++ b/tests/hooks/test_verify_local.sh @@ -1611,6 +1611,15 @@ else printf '%s\n' "$fake_pre_push_provider_contract_output" fi +fake_pre_push_executable_operation_output="$(run_fake_verify pre-push crates/warp-core/tests/executable_operation_pipeline_tests.rs)" +fake_pre_push_executable_operation_cargo_log="$(extract_log_section cargo-log "$fake_pre_push_executable_operation_output")" +if printf '%s\n' "$fake_pre_push_executable_operation_cargo_log" | grep -q -- 'test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests'; then + pass "pre-push keeps required scheduler Action features for the exact integration test" +else + fail "pre-push should keep required scheduler Action features for executable_operation_pipeline_tests" + printf '%s\n' "$fake_pre_push_executable_operation_output" +fi + fake_pre_push_installed_contract_output="$(run_fake_verify pre-push crates/warp-core/tests/installed_contract_registry_tests.rs)" fake_pre_push_installed_contract_cargo_log="$(extract_log_section cargo-log "$fake_pre_push_installed_contract_output")" if printf '%s\n' "$fake_pre_push_installed_contract_cargo_log" | grep -q -- 'test -p warp-core --features native_rule_bootstrap --test installed_contract_registry_tests'; then From 242392d76eeccb70c28dd075a7d7d51bad9a323c Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 04:59:09 -0700 Subject: [PATCH 03/64] docs: define scheduler-owned executable Action lifecycle --- CHANGELOG.md | 23 ++- README.md | 18 +- docs/README.md | 5 + ...-admitted-executable-operation-packages.md | 8 + ...uler-owned-executable-operation-actions.md | 185 ++++++++++++++++++ docs/adr/README.md | 1 + .../application-contract-hosting.md | 39 ++-- docs/spec/scheduler-warp-core.md | 31 ++- docs/topics/GeneratedRules.md | 10 +- docs/topics/WAL.md | 28 ++- 10 files changed, 309 insertions(+), 39 deletions(-) create mode 100644 docs/adr/0025-scheduler-owned-executable-operation-actions.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c7c4714b..956adb26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,30 @@ ### Added +- Executable-operation application writes now enter Echo as canonical, + WAL-acknowledged Actions and are evaluated only by the scheduler while + constructing a Tick (ADR 0025). Accepted pre-Tick Actions recover as pending + work. Two independent Actions can share one exact parent coordinate and + contribute to one composite Tick, while retaining candidate-specific + application-basis propositions and per-Action typed outcomes. Footprint + conflicts name earlier applied members; evaluator obstructions contribute no + operations. One scheduler WAL transaction retains each Action's receipt + correlation and typed outcome in canonical order followed by exactly one + replayable state delta. The decided Tick is durable before state, frontier, + receipt, or outcome publication, and fresh-host recovery validates every + outcome against its exact envelope, invocation, installed operation, Tick + entry, composite consequence, and state root. Direct operation + prepare/commit remains a hidden transitional compatibility/test seam rather + than an application execution lifecycle. - `TrustedRuntimeHost` now has the first hook-free executable-operation runtime slice. A runtime owner can admit exact canonical `ExecutableOperationPackageV1` bytes under a separate package policy, install their data-only `EchoOperationProgramV1`, independently admit an exact-basis invocation under caller authority and delegated budget, evaluate privately, and either commit one parent-visible patch or return typed noncommit evidence. - Only committed operation consequences enter the operation-tick WAL. The initial - generic program performs an anchored typed-node alpha-attachment + On that transitional direct seam, only committed operation consequences + enter the operation-tick WAL. The initial generic program performs an + anchored typed-node alpha-attachment compare-and-set; it contains no application matcher, executor, footprint callback, or prebuilt mutation plan. Package, installation, invocation, evaluation, actual-footprint, budget, patch, result, basis, and terminal @@ -28,8 +44,7 @@ callbacks. A program digest alone cannot install, invoke, or authorize an operation. This slice does not yet include Edict compiler emission, a structurally separate target verifier, Jedit's rope - lawpack, `ReplaceRange`, scheduler batch composition, or an independently - implemented semantic oracle. + lawpack, `ReplaceRange`, or an independently implemented semantic oracle. - The executable-operation corridor now has a separate `AnchoredNodeAttachmentCreateIfAbsent` program (ADR 0024). The original compare-and-set program remains update-only with its canonical program, diff --git a/README.md b/README.md index 4cad6add..106167fa 100644 --- a/README.md +++ b/README.md @@ -248,17 +248,23 @@ The current executable-operation runtime slice is: canonical ExecutableOperationPackageV1 bytes -> Echo-owned package and invocation admission -> installed data-only EchoOperationProgramV1 --> bounded private Echo evaluation --> exact-basis singleton commit --> typed receipt, WAL retention, and callback-free recovery +-> canonical Action submission retained before acknowledgement +-> runtime-owned admission into the ordinary head inbox +-> scheduler selection at one exact basis +-> bounded private Echo evaluation during Tick construction +-> one composite Tick consequence with typed per-Action outcomes +-> decided Tick WAL retention before state, frontier, and Receipt publication +-> callback-free pending-Action and decided-Tick recovery ``` The first two paths are callback-shaped compatibility infrastructure. The third proves Echo-owned execution of admitted data-only meaning through separate update-only compare-and-set and single-node create-if-absent program -profiles. No real Edict compiler output, Jedit operation, or Graft -multi-record mutation uses them yet. The next convergence crossing must bind a -real application-owned Edict operation and lawpack to the executable-operation +profiles. Independent Actions for one head can share one scheduler-owned Tick; +conflicting or obstructed Actions retain typed outcomes without hidden +mutation. No real Edict compiler output, Jedit operation, or Graft operation +uses this route yet. The next convergence crossing must bind a real +application-owned Edict operation and lawpack to the executable-operation package without reintroducing a native implementation. ## Contracts And Boundaries diff --git a/docs/README.md b/docs/README.md index beec44e4..4562084c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ causal history. Git history is the archive; GitHub owns live work and status. - [Local contract host quickstart](quickstart-local-contract-host.md) - [Echo 1.0 release contract](releases/echo-1.0-contract.md) - [WARP core runtime](spec/warp-core.md) +- [Scheduler WARP core](spec/scheduler-warp-core.md) ## Current Architecture @@ -51,6 +52,10 @@ causal history. Git history is the archive; GitHub owns live work and status. - [Bunny owns reusable geometry](adr/0019-bunny-owns-reusable-geometry.md) - [Retained reading storage and proof boundary](adr/0020-retained-reading-storage-and-proof-boundary.md) - [Public optic and observation boundary](adr/0021-public-optic-observation-boundary.md) +- [Application-requested causal-anchor admission](adr/0022-application-requested-causal-anchor-admission.md) +- [Admitted executable-operation packages](adr/0023-admitted-executable-operation-packages.md) +- [Anchored-node creation from absence](adr/0024-anchored-node-creation-from-absence.md) +- [Scheduler-owned executable-operation Actions](adr/0025-scheduler-owned-executable-operation-actions.md) ## Normative Contracts diff --git a/docs/adr/0023-admitted-executable-operation-packages.md b/docs/adr/0023-admitted-executable-operation-packages.md index cffb21fa..07e40dd1 100644 --- a/docs/adr/0023-admitted-executable-operation-packages.md +++ b/docs/adr/0023-admitted-executable-operation-packages.md @@ -6,6 +6,14 @@ - **Status:** Accepted - **Date:** 2026-07-18 - **Partially supersedes:** ADR 0015 for newly authored executable operations +- **Amended by:** ADR 0025 for scheduler-owned Action execution + +> **Scheduler lifecycle amendment:** This ADR established exact package, +> invocation, private-evaluation, and recovery semantics. ADR 0025 supersedes +> direct prepare/commit as the application lifecycle: canonical executable +> Actions now enter ordinary durable ingress, and only the scheduler invokes +> private evaluation while constructing an atomic Tick. The direct methods are +> transitional compatibility and test seams. ## Context diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md new file mode 100644 index 00000000..1d4b1856 --- /dev/null +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -0,0 +1,185 @@ + + + +# ADR 0025: Scheduler-Owned Executable-Operation Actions + +- **Status:** Accepted +- **Date:** 2026-07-24 +- **Amends:** ADR 0023 + +## Context + +ADR 0023 introduced Echo-interpreted executable-operation packages so +application semantics no longer need a native callback. Its first runtime +crossing nevertheless exposes a second execution lifecycle: + +```text +admit invocation +prepare privately +commit prepared operation directly +``` + +That crossing proved exact package resolution, bounded private evaluation, +typed obstruction, patch validation, and durable recovery. It is not the +permanent application route. Echo's public write lifecycle is an `Action` +accepted into causal ingress, selected by the scheduler, and decided as part of +one atomic `Tick`. + +Keeping direct operation commit as a peer lifecycle would let application or +host-adapter code choose when evaluation runs, bypass normal Lane/head +selection, and publish a singleton consequence outside the scheduler's +multi-Action decision. That would preserve the same callback-shaped authority +problem Edict was introduced to remove, merely behind a different artifact. + +The current scheduler and WAL also assume different cardinalities: + +- one head commit may decide several ingress envelopes; +- `TrustedRuntimeHost::tick_once` writes one scheduler-WAL transaction per + receipt correlation; +- the filesystem WAL rejects more than one such transaction because the group + would not be failure-atomic. + +One Tick must be one durable transaction, regardless of its Action count. + +## Decision + +### Executable operations enter through canonical Actions + +Echo reserves one stable `IntentKind` for executable-operation Actions. Its +payload is exactly one canonical `EchoOperationInvocationV1`; package meaning +is never copied into the envelope and no native callback is attached. + +Application-facing code may only submit that envelope through the ordinary +submission boundary. An acknowledgement means the canonical Action and its +acceptance evidence have committed to the WAL. It does not mean the Action has +executed. + +The runtime owner configures invocation-admission policy. After acceptance, +Echo resolves the exact installed package, issues ticketed runtime ingress, and +places the Action in the normal target head inbox. A restart reconstructs the +accepted pending Action from retained submission material; admission and +staging are deterministic and repeatable. + +### The scheduler owns evaluation + +Only scheduler Tick construction invokes the private bounded evaluator for an +Action. The evaluator remains private and pure with respect to the parent +worldline: + +- success returns one complete prepared candidate; +- obstruction returns typed evidence and no parent-visible operations; +- neither result mutates the parent state. + +The direct `prepare_echo_operation_v1` and +`commit_prepared_echo_operation_v1` methods remain temporarily available only +as explicitly transitional compatibility/test seams. They are not the +application architecture and will be removed after existing convergence +witnesses migrate. + +### Legacy and executable ingress never share an evaluator batch + +Provider/native ingress and executable-operation Actions are different +execution categories. A head inbox may contain both while migrations are in +flight, but one Tick candidate batch is homogeneous. + +The scheduler examines the lowest canonical ingress identity and drains only +that entry's execution category, subject to the existing inbox budget. Other +categories remain pending for a later Tick. This is deterministic, preserves +global inbox order at category boundaries, and makes it impossible for an +executable Action to fall through to a native callback engine. + +### One exact parent basis, per-Action application propositions + +Actions selected into one Tick share the exact Echo parent coordinate: + +- writer head; +- worldline tick; +- committing global-tick predecessor; +- state root; +- parent commit identity. + +Each Action retains its own application-basis proposition. Two independent +Actions over different domain values therefore need not have byte-identical +complete `EchoOperationEvaluationBasisV1` values. This amends ADR 0023's +informal statement that composed preparations share identical complete basis +bytes: they share the runtime parent fields, while the separately typed +application proposition remains candidate-specific. + +### Tick construction is atomic + +Prepared candidates are considered in canonical Action order. The scheduler +reserves their actual footprints against already accepted candidates: + +- independent candidates are applied to a private successor state; +- footprint conflicts produce a rejected Action outcome naming earlier + applied blockers; +- evaluator obstructions produce a typed obstructed Action outcome with no + blockers and no mutation. + +All accepted candidate operations are canonicalized into one +`WarpTickPatchV1`. Tick construction produces one snapshot, one Tick receipt +with one entry per Action, one provenance entry, and one worldline frontier +advance. Failure while evaluating, composing, applying, or validating discards +the private successor in full. + +The generic Tick receipt classifies scheduler disposition. A separate typed +executable-Action outcome binds the exact invocation, preparation or +obstruction, Tick identity, and committed member consequence. Application +semantics are not inferred from a generic receipt label. + +### One Tick is one WAL transaction + +A scheduler Tick WAL transaction contains, in canonical Action order: + +```text +TickReceiptRecorded + ReceiptCorrelationRecorded + ActionOutcomeRecorded +... repeated for each Action ... +RuntimeStateDeltaRecorded +``` + +Every receipt record cites the same Tick receipt content digest and the +submission-specific causal coordinate. The state delta appears exactly once +because the Tick has exactly one atomic state consequence. + +The WAL transaction is committed before the live runtime, frontier, provenance, +Action outcomes, or receipts are published. On append failure, live state is +restored to the accepted-pending posture. Recovery validates the whole group, +reconstructs the state transition once, and restores every per-Action outcome +and receipt correlation. + +## Consequences + +- Application code cannot invoke executable evaluation or direct commit. +- Two independent executable-operation Actions can contribute to one + scheduler-owned Tick. +- A typed obstruction is durable evidence and cannot hide a state mutation. +- Filesystem durability no longer depends on several scheduler transactions + being atomically appended as an external batch. +- Existing executable package, program, invocation, and direct singleton + receipt bytes remain unchanged. +- Native/provider callback ingress remains compatibility infrastructure and is + not extended by this decision. + +## Acceptance + +The implementation is accepted only with executable witnesses that prove: + +1. Action acknowledgement follows WAL commit. +2. Accepted pre-Tick Actions survive restart as pending work. +3. Submission and runtime admission do not evaluate or mutate state. +4. Only scheduler Tick construction invokes private evaluation. +5. Two independent Actions produce one Tick and one atomic state consequence. +6. Tick construction or WAL failure publishes no partial state or receipt. +7. Typed obstruction contributes no operations. +8. The decided Tick is durable before frontier and receipt publication. +9. Fresh-host recovery reconstructs Action outcomes, Tick, state, and receipts. +10. Direct prepare/commit is marked transitional and absent from the + application-facing surface. + +## Non-Goals + +- Real Edict compiler emission or a production Graft lawpack. +- Provider-v1 or native callback generalization. +- Arbitrary multi-record program semantics. +- Cross-head or cross-worldline atomic Ticks. +- Migration from Graft's git-warp database. diff --git a/docs/adr/README.md b/docs/adr/README.md index 170b0ecf..4ca1995d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -46,6 +46,7 @@ track work, progress, priority, or release readiness. | [0022](0022-application-requested-causal-anchor-admission.md) | Accepted | Application-requested, Echo-owned anchor admission | | [0023](0023-admitted-executable-operation-packages.md) | Accepted | Admitted executable operation packages | | [0024](0024-anchored-node-creation-from-absence.md) | Accepted | Anchored-node creation as a separate executable program | +| [0025](0025-scheduler-owned-executable-operation-actions.md) | Accepted | Scheduler-owned executable-operation Actions | ADR 0006 predates this index contract and did not declare a status. Its superseded tombstone preserves that fact without silently ratifying the old diff --git a/docs/architecture/application-contract-hosting.md b/docs/architecture/application-contract-hosting.md index 9824987d..4770ab0f 100644 --- a/docs/architecture/application-contract-hosting.md +++ b/docs/architecture/application-contract-hosting.md @@ -151,23 +151,28 @@ accepts a distinct executable-operation category for new application mutations. Echo now implements two bounded anchored-attachment programs in that runtime slice: the original update-only compare-and-set and a separately identified single-node-plus-alpha-attachment create-if-absent profile. Package -and invocation admission, private bounded evaluation, exact-basis singleton -commit, typed evidence, WAL, and callback-free fresh-host recovery validate the -selected program's exact footprint, budget, result, and patch shape. -Operations targeting a descended WARP retain and meter every portal attachment -in the root-to-target reachability chain as a causal input. -Noncommitted evidence is returned to the caller but does not enter the -operation-tick WAL. The slice exposes no application matcher, executor, or -footprint callback. It is not yet emitted by Edict, does not implement Jedit -`ReplaceRange` or Graft's multi-record mutation, and has no structurally -separate target verifier or general scheduler batch composition. Provider v1 -remains stable while consumers migrate; it is not renamed or silently -reinterpreted as the executable-operation corridor. The program digest -supplies executable meaning only: it cannot independently confer an operation -coordinate, invocability, or authority, and Echo cannot install or invoke it -naked. The admitted operation package binds the public contract and semantic -closure to the exact program, after which Echo independently admits each -invocation. +and invocation admission, WAL-durable Action intake, ordinary head-inbox +staging, scheduler-owned bounded evaluation, exact-basis multi-Action Tick +composition, typed per-Action outcomes, and callback-free fresh-host recovery +validate each selected program's exact footprint, budget, result, and patch +shape. Operations targeting a descended WARP retain and meter every portal +attachment in the root-to-target reachability chain as a causal input. +Scheduler-selected obstructions and footprint conflicts enter the decided Tick +as typed no-hidden-mutation outcomes. The decided Tick reaches the WAL before +state, frontier, or Receipt publication. The transitional direct +prepare/commit seam can still return noncommitted evidence to trusted host +tests, but it is not the application lifecycle. + +The slice exposes no application matcher, executor, or footprint callback. It +is not yet emitted by Edict, does not implement Jedit `ReplaceRange` or any +Graft operation, has no structurally separate target verifier, and does not +claim cross-head atomic filesystem-WAL persistence. Provider v1 remains stable +while consumers migrate; it is not renamed or silently reinterpreted as the +executable-operation corridor. The program digest supplies executable meaning +only: it cannot independently confer an operation coordinate, invocability, or +authority, and Echo cannot install or invoke it naked. The admitted operation +package binds the public contract and semantic closure to the exact program, +after which Echo independently admits each invocation. ## External Edict Provider Artifacts diff --git a/docs/spec/scheduler-warp-core.md b/docs/spec/scheduler-warp-core.md index 80b4046d..9eb7c8ce 100644 --- a/docs/spec/scheduler-warp-core.md +++ b/docs/spec/scheduler-warp-core.md @@ -10,6 +10,7 @@ Depends on: - [Canonical Inbox Sequencing](canonical-inbox-sequencing.md) - [SPEC-0003 - DPO Concurrency Litmus v0](SPEC-0003-dpo-concurrency-litmus-v0.md) - [WARP Tick Patch](warp-tick-patch.md) +- [ADR 0025 - Scheduler-Owned Executable-Operation Actions](../adr/0025-scheduler-owned-executable-operation-actions.md) ## Purpose @@ -36,10 +37,12 @@ Reservation checks all candidate resources for conflict and marks resources only A candidate conflicts when it writes a resource another admitted candidate reads or writes, or when its boundary port claim overlaps another admitted boundary port claim. Reads may overlap reads. The broad WARP outcome algebra is `Derived | Plural | Conflict | -Obstruction`. Echo's local `TickReceipt` entries currently realize the narrower -tick-scale shape `Applied / Rejected(FootprintConflict)`, with blocker -attribution for conflicts. `Plural` belongs to broader braid/replica-scale work -until an executable local claim requires it. +Obstruction`. Echo's local `TickReceipt` entries realize the narrower +tick-scale shape `Applied / Rejected(FootprintConflict | +ExecutableOperationObstruction)`. Footprint conflicts name earlier applied +blockers. An executable-operation obstruction has no blockers and contributes +no mutation. `Plural` belongs to broader braid/replica-scale work until an +executable local claim requires it. Conflict rejection is final for that tick attempt. Retry is a new explicit causal act, not a hidden retry queue. @@ -56,3 +59,23 @@ Pending rewrites drain in lexicographic order derived from `scope_hash`, stable ## Decision 5: Radix reservation is the default implementation The default scheduler uses generation-stamped active sets for membership checks. Expected reservation cost is proportional to candidate footprint size. + +## Decision 6: Executable Actions are evaluated only inside Tick construction + +A canonical executable-operation Action enters the same durable submission and +head-inbox lifecycle as other work. The runtime resolves the exact installed +package and admission policy, but neither submission nor admission evaluates +application semantics. + +The scheduler partitions executable and legacy/native ingress deterministically +at the lowest canonical ingress id so the two evaluator categories never share +one candidate batch. Inside an executable batch, Action ingress order is +canonical. Private bounded evaluation yields either a complete prepared +candidate or a typed no-mutation obstruction. The scheduler reserves successful +candidate footprints, constructs one composite consequence, and emits one Tick +receipt entry per Action. + +The successor state remains private until the complete Tick transaction is +durable. Construction failure discards it. WAL failure restores the +accepted-pending posture. The direct executable-operation prepare/commit methods +are transitional host/test seams, not a second application execution path. diff --git a/docs/topics/GeneratedRules.md b/docs/topics/GeneratedRules.md index 67175756..d8208a79 100644 --- a/docs/topics/GeneratedRules.md +++ b/docs/topics/GeneratedRules.md @@ -140,11 +140,13 @@ exact canonical ExecutableOperationPackageV1 bytes -> Echo-owned package admission under a separate policy -> installed data-only EchoOperationProgramV1 -> exact basis-bearing canonical invocation +-> canonical Action submission retained before acknowledgement -> Echo-owned invocation admission under authority and delegated budget --> bounded private evaluation with recorded actual footprint --> exact-basis singleton commit attempt --> one committed TickPatch and typed receipt, or returned typed no-patch evidence --> committed outcomes only: execution-kernel WAL retention and callback-free recovery +-> ordinary head-inbox staging and scheduler selection at one exact basis +-> bounded private evaluation with recorded actual footprint during Tick construction +-> deterministic composition of independent Actions into one Tick consequence +-> typed committed, obstructed, or footprint-conflict outcome for every Action +-> decided Tick WAL retention before publication and callback-free recovery ``` The first two programs are an anchored typed-node alpha-attachment diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index 5a1348c9..5ade6c24 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -17,7 +17,7 @@ Echo may only claim what its WAL can recover. ## What We Found -The current runtime WAL evidence says nine concrete things. +The current runtime WAL evidence says ten concrete things. First, accepted-submission evidence is not just an in-memory editor event. The WAL-backed ACK path, `submit_intent_with_runtime_wal_ack(...)`, returns only @@ -89,9 +89,13 @@ state-delta record is inspectable but obstructs writable startup. Transaction construction also requires the receipt and correlation to name the same causal event and the retained state delta to bind that receipt's content commitment before the transaction can commit. -Each scheduler transaction must also contain exactly one receipt, correlation, -and runtime state-delta frame; recovery rejects duplicate claims rather than -selecting whichever frame appears first. +Legacy singleton scheduler transactions contain exactly one receipt, +correlation, and runtime state-delta frame. Scheduler-owned +executable-operation Action Ticks instead retain one ordered receipt, +correlation, and typed Action-outcome group per selected Action followed by +exactly one runtime state delta for the whole Tick. Recovery rejects missing, +duplicated, reordered, mixed, or cross-Tick claims rather than selecting +whichever frame appears first. WAL activation is non-lossy: if the live host contains submissions, staged ingress, receipt correlations, provenance, pending inbox work, cycle progress, or worldline state that recovered WAL evidence cannot reproduce, activation @@ -127,6 +131,22 @@ or mis-scoped frame/frontier shapes. These records do not make a naked program digest invocable and do not persist the process-local authority needed to commit an unretained preparation. +Tenth, canonical executable-operation Actions use the ordinary WAL-backed +submission acknowledgement before execution. A crash after acknowledgement and +before scheduler selection restores accepted pending work. Runtime-owned +admission resolves the exact installed package, but only scheduler Tick +construction invokes private bounded evaluation. Two independent Actions can +produce one composite Tick with one snapshot, one provenance entry, one +frontier advance, per-Action typed outcomes, and one replayable state delta. + +The scheduler transaction commits before state, frontier, receipt, or Action +outcome publication. Construction and WAL failures publish none of them. A +typed evaluator obstruction is retained as a lawful no-mutation Tick entry. +Fresh-host recovery binds each Action outcome back to its exact retained +envelope, invocation identity, installed operation, scope, Tick entry, +disposition, blocker set, composite patch, and final state root. Swapped or +misclassified outcome evidence therefore fails closed. + ## Boundaries The WAL belongs to the trusted runtime host. Application-facing code can submit From 75a8514784719032b0e87625d2752dac36ce1c80 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:10:42 -0700 Subject: [PATCH 04/64] fix: register scheduler Action ADR namespace --- tests/docs/test_adr_namespace.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/docs/test_adr_namespace.sh b/tests/docs/test_adr_namespace.sh index 45417e27..d0b07989 100755 --- a/tests/docs/test_adr_namespace.sh +++ b/tests/docs/test_adr_namespace.sh @@ -35,9 +35,10 @@ readonly current_adrs=( "0022-application-requested-causal-anchor-admission.md" "0023-admitted-executable-operation-packages.md" "0024-anchored-node-creation-from-absence.md" + "0025-scheduler-owned-executable-operation-actions.md" ) -readonly current_adr_last=24 +readonly current_adr_last=25 readonly superseded_legacy_adrs=( "ADR-0003-Materialization-Bus.md" From 0bd7d9fd17cdf9a7126c77aa57779720d8bb9994 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:16:15 -0700 Subject: [PATCH 05/64] fix: render executable operation obstructions --- xtask/src/hello_echo.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xtask/src/hello_echo.rs b/xtask/src/hello_echo.rs index b341b913..bb6eb11b 100644 --- a/xtask/src/hello_echo.rs +++ b/xtask/src/hello_echo.rs @@ -679,6 +679,12 @@ fn receipt_report(receipt: &TickReceipt) -> ReceiptReport { conflict_count += 1; "rejected:footprint-conflict" } + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) => { + rejected_count += 1; + "rejected:executable-operation-obstruction" + } }; entries.push(ReceiptEntryReport { index, From dff7284cec27fa9b5333875cf4d065f7b4f5a5ee Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:32:03 -0700 Subject: [PATCH 06/64] Fix: defer executable Action basis checks to Tick evaluation --- crates/warp-core/src/echo_operation.rs | 121 ++++++++++++++---- crates/warp-core/src/trusted_runtime_host.rs | 25 ++-- .../executable_operation_pipeline_tests.rs | 107 +++++++++++++++- 3 files changed, 215 insertions(+), 38 deletions(-) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index a5a29fb2..a8151237 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -2307,6 +2307,85 @@ pub(crate) fn admit_invocation_v1( current_state: &WorldlineState, evaluation_authority: EchoOperationEvaluationAuthorityV1, ) -> Result { + let (invocation, installed) = + admit_invocation_static_v1(installed, policy, canonical_invocation_bytes)?; + if invocation.evaluation_basis != current_basis { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation does not name the current exact parent basis", + )); + } + if invocation + .evaluation_basis + .application_basis + .schema_identity + != installed.application_basis_schema_identity + { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation application-basis schema differs from installed package", + )); + } + let current_application_basis = + current_application_basis(installed, &invocation, current_state)?; + if invocation.evaluation_basis.application_basis != current_application_basis { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation application basis differs from Echo's current graph proposition", + )); + } + Ok(build_admitted_invocation_v1( + invocation, + installed, + policy, + canonical_invocation_bytes, + evaluation_authority, + )) +} + +/// Admits the static installed meaning and runtime-owned authority for one +/// scheduler-bound executable-operation Action. +/// +/// Exact basis freshness is deliberately deferred to private evaluation during +/// Tick construction. That lets an Action which was durably accepted at an +/// earlier basis reach the evaluator and receive a typed `BasisChanged` +/// obstruction instead of failing the host admission loop. +pub(crate) fn admit_action_invocation_v1( + installed: Option<&InstalledEchoOperationV1>, + policy: EchoOperationInvocationAdmissionPolicyV1, + canonical_invocation_bytes: &[u8], + evaluation_authority: EchoOperationEvaluationAuthorityV1, +) -> Result { + let (invocation, installed) = + admit_invocation_static_v1(installed, policy, canonical_invocation_bytes)?; + if invocation + .evaluation_basis + .application_basis + .schema_identity + != installed.application_basis_schema_identity + { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation application-basis schema differs from installed package", + )); + } + Ok(build_admitted_invocation_v1( + invocation, + installed, + policy, + canonical_invocation_bytes, + evaluation_authority, + )) +} + +fn admit_invocation_static_v1<'a>( + installed: Option<&'a InstalledEchoOperationV1>, + policy: EchoOperationInvocationAdmissionPolicyV1, + canonical_invocation_bytes: &[u8], +) -> Result< + (EchoOperationInvocationV1, &'a InstalledEchoOperationV1), + EchoOperationInvocationAdmissionErrorV1, +> { let invocation = EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) .map_err(|error| { invocation_admission_error( @@ -2360,38 +2439,24 @@ pub(crate) fn admit_invocation_v1( "delegated budget is below the program minimum or exceeds an admitted ceiling", )); } - if invocation.evaluation_basis != current_basis { - return Err(invocation_admission_error( - EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, - "invocation does not name the current exact parent basis", - )); - } - if invocation - .evaluation_basis - .application_basis - .schema_identity - != installed.application_basis_schema_identity - { - return Err(invocation_admission_error( - EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, - "invocation application-basis schema differs from installed package", - )); - } - let current_application_basis = - current_application_basis(installed, &invocation, current_state)?; - if invocation.evaluation_basis.application_basis != current_application_basis { - return Err(invocation_admission_error( - EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, - "invocation application basis differs from Echo's current graph proposition", - )); - } + Ok((invocation, installed)) +} + +fn build_admitted_invocation_v1( + invocation: EchoOperationInvocationV1, + installed: &InstalledEchoOperationV1, + policy: EchoOperationInvocationAdmissionPolicyV1, + canonical_invocation_bytes: &[u8], + evaluation_authority: EchoOperationEvaluationAuthorityV1, +) -> AdmittedEchoOperationInvocationV1 { let invocation_id = EchoOperationInvocationIdV1(domain_hash( INVOCATION_ID_DOMAIN, canonical_invocation_bytes, )); let admission_policy_id = policy.identity(); let installed_operation_id = installed.installed_operation_id; - Ok(AdmittedEchoOperationInvocationV1 { + let evaluation_basis_id = invocation.evaluation_basis.identity(); + AdmittedEchoOperationInvocationV1 { invocation, invocation_id, canonical_invocation_bytes: canonical_invocation_bytes.to_vec(), @@ -2400,12 +2465,12 @@ pub(crate) fn admit_invocation_v1( installed_operation_id, invocation_id, admission_policy_id, - current_basis.identity(), + evaluation_basis_id, ), installed_operation_id, evaluation_authority, admission_policy: policy, - }) + } } fn current_application_basis( diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 364e8390..c2f93242 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -41,14 +41,15 @@ use crate::{ }, contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ - admit_invocation_v1, admit_package_v1, commit_prepared_to_state, - decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, - inspect_action_invocation_v1, install_recovered_v1, installed_from_admitted, - not_committed_basis_changed, not_committed_evaluation_authority_mismatch, - not_committed_installation_unavailable, operation_descent_stack, prepare_operation_v1, - recover_action_outcome_v1, recover_committed_execution_receipt_v1, recover_installation_v1, - retain_action_outcome_v1, retain_committed_execution_v1, retain_installation_v1, - validate_receipt_installation_v1, EchoOperationEvaluationAuthorityV1, + admit_action_invocation_v1, admit_invocation_v1, admit_package_v1, + commit_prepared_to_state, decode_invocation_route_v1, + echo_operation_action_invocation_bytes_v1, inspect_action_invocation_v1, + install_recovered_v1, installed_from_admitted, not_committed_basis_changed, + not_committed_evaluation_authority_mismatch, not_committed_installation_unavailable, + operation_descent_stack, prepare_operation_v1, recover_action_outcome_v1, + recover_committed_execution_receipt_v1, recover_installation_v1, retain_action_outcome_v1, + retain_committed_execution_v1, retain_installation_v1, validate_receipt_installation_v1, + EchoOperationEvaluationAuthorityV1, }, provider_contract::admit_provider_contract_package_v1, AdmittedEchoOperationInvocationV1, AdmittedExecutableOperationPackageV1, @@ -1600,7 +1601,13 @@ impl TrustedRuntimeHost { .ok_or(TrustedRuntimeHostError::EchoOperationActionAdmissionPolicyUnavailable)?; let mut admitted_actions = BTreeMap::new(); for (submission, envelope, invocation_bytes) in pending { - let admitted = self.admit_echo_operation_invocation_v1(&policy, &invocation_bytes)?; + let (package_id, _) = decode_invocation_route_v1(&invocation_bytes)?; + let admitted = admit_action_invocation_v1( + self.engine.installed_echo_operation_package_v1(package_id), + policy, + &invocation_bytes, + self.echo_operation_evaluation_authority.clone(), + )?; let admission_digest = echo_operation_action_admission_digest(&submission, &admitted); self.runtime.ingest_echo_operation_action_v1( &TicketedRuntimeIngressAuthority::assume_runtime_owner(), diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 82f61737..b97c7243 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -169,6 +169,20 @@ fn fixture_host_with_bare_node( second_node_local_id: warp_core::NodeId, second_node_type: warp_core::TypeId, second_attachment: Option<(warp_core::TypeId, &'static [u8])>, +) -> (TrustedRuntimeHost, WriterHeadKey, NodeKey, NodeKey) { + fixture_host_with_bare_node_and_policy( + second_node_local_id, + second_node_type, + second_attachment, + InboxPolicy::AcceptAll, + ) +} + +fn fixture_host_with_bare_node_and_policy( + second_node_local_id: warp_core::NodeId, + second_node_type: warp_core::TypeId, + second_attachment: Option<(warp_core::TypeId, &'static [u8])>, + inbox_policy: InboxPolicy, ) -> (TrustedRuntimeHost, WriterHeadKey, NodeKey, NodeKey) { let warp_id = make_warp_id("operation-fixture"); let node_id = make_node_id("operation-fixture-root"); @@ -221,7 +235,7 @@ fn fixture_host_with_bare_node( .register_writer_head(WriterHead::with_routing( head_key, PlaybackMode::Play, - InboxPolicy::AcceptAll, + inbox_policy, None, true, )) @@ -2575,6 +2589,97 @@ fn update_precondition_refuses_when_the_node_has_the_wrong_type() { ); } +#[test] +fn budget_deferred_executable_action_reaches_typed_basis_obstruction() { + let second_node_id = make_node_id("operation-fixture-budget-deferred"); + let attachment_type = make_type_id("operation-fixture-atom"); + let (mut host, head_key, first_node, second_node) = fixture_host_with_bare_node_and_policy( + second_node_id, + make_type_id("operation-fixture-node"), + Some((attachment_type, b"second-before")), + InboxPolicy::Budgeted { max_per_tick: 1 }, + ); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let first = action_invocation( + &host, + &installed, + head_key, + first_node, + b"before", + b"first-after", + ); + let second = action_invocation( + &host, + &installed, + head_key, + second_node, + b"second-before", + b"second-after", + ); + let first_submission = host + .app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + first, + ) + .expect("the first invocation is one canonical Action"), + ) + .expect("the first Action is accepted"); + let second_submission = host + .app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + second, + ) + .expect("the second invocation is one canonical Action"), + ) + .expect("the second Action is accepted"); + let first_submission_id = first_submission.submission_id; + let second_submission_id = second_submission.submission_id; + + let first_tick = host + .tick_once() + .expect("one Action is selected at the original basis"); + assert_eq!(first_tick.len(), 1); + assert_eq!(first_tick[0].admitted_count, 1); + + let (committed_submission_id, deferred_submission_id) = match ( + host.echo_operation_action_outcome_v1(&first_submission_id), + host.echo_operation_action_outcome_v1(&second_submission_id), + ) { + (Some(EchoOperationActionOutcomeV1::Committed(_)), None) => { + (first_submission_id, second_submission_id) + } + (None, Some(EchoOperationActionOutcomeV1::Committed(_))) => { + (second_submission_id, first_submission_id) + } + outcomes => panic!("exactly one Action must commit in the first Tick: {outcomes:?}"), + }; + + let second_tick = host + .tick_once() + .expect("the deferred Action reaches scheduler-owned private evaluation"); + assert_eq!(second_tick.len(), 1); + assert_eq!(second_tick[0].admitted_count, 1); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&deferred_submission_id) + else { + panic!("the stale deferred Action must receive one typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); + assert!(matches!( + host.echo_operation_action_outcome_v1(&committed_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + #[test] fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { let wal_dir = TempWalDir::new(); From 6043924c2aee4fec3dbb8f70a3cce4585c766b44 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:34:59 -0700 Subject: [PATCH 07/64] Fix: index pending Action submissions by decision state --- crates/warp-core/src/coordinator.rs | 47 ++++++++++++++++++ crates/warp-core/src/trusted_runtime_host.rs | 48 ++++++++----------- .../executable_operation_pipeline_tests.rs | 6 +++ 3 files changed, 72 insertions(+), 29 deletions(-) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 925155ff..5bb29d9b 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -1141,6 +1141,11 @@ pub struct WorldlineRuntime { next_submission_generation: IngressSubmissionGeneration, /// Witnessed submission records keyed by content-addressed submission id. witnessed_submissions: BTreeMap, + /// Undecided witnessed submission ids. + /// + /// This is a derived, recovery-rebuilt scheduler index. It keeps pending + /// admission proportional to outstanding work instead of retained history. + pending_witnessed_submission_ids: BTreeSet, /// Canonical envelope material for witnessed submissions. witnessed_submission_envelopes: BTreeMap, /// Deterministic lookup from resolved semantic target and ingress id to @@ -1203,6 +1208,7 @@ struct ReceiptCorrelationRollbackEntry { previous_receipt_ref_ticketed_ingress: Option, current_basis: (WorldlineId, WorldlineTick, Hash), previous_current_basis_ticketed_ingresses: Option>, + previous_pending_submission: bool, } #[derive(Clone, Debug, Default)] @@ -1325,6 +1331,13 @@ impl WorldlineRuntime { .remove(&entry.current_basis); } } + if entry.previous_pending_submission { + self.pending_witnessed_submission_ids + .insert(entry.submission_id); + } else { + self.pending_witnessed_submission_ids + .remove(&entry.submission_id); + } } } @@ -1379,6 +1392,22 @@ impl WorldlineRuntime { self.witnessed_submissions.values() } + /// Iterates only undecided witnessed submissions in deterministic id order. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn pending_witnessed_submissions( + &self, + ) -> impl Iterator { + self.pending_witnessed_submission_ids + .iter() + .filter_map(|submission_id| self.witnessed_submissions.get(submission_id)) + } + + /// Returns the number of witnessed submissions without a receipt decision. + #[must_use] + pub fn pending_witnessed_submission_count(&self) -> usize { + self.pending_witnessed_submission_ids.len() + } + /// Returns witnessed submissions in deterministic replay order. /// /// Replay order follows Echo-owned submission generation, then submission id @@ -2140,6 +2169,16 @@ impl WorldlineRuntime { self.next_submission_generation = next_submission_generation; self.submission_by_target .extend(staged_submission_by_target); + self.pending_witnessed_submission_ids.extend( + staged_witnessed_submissions + .keys() + .filter(|submission_id| { + !self + .receipt_correlation_by_submission + .contains_key(*submission_id) + }) + .copied(), + ); self.witnessed_submissions .extend(staged_witnessed_submissions); Ok(()) @@ -2430,6 +2469,8 @@ impl WorldlineRuntime { .entry(current_basis) .or_default() .insert(ticketed_ingress_id); + self.pending_witnessed_submission_ids + .remove(&persisted.submission_id); self.worldlines .frontier_mut(&persisted.head_key.worldline_id) .ok_or(RuntimeError::UnknownWorldline( @@ -2835,6 +2876,7 @@ impl WorldlineRuntime { .insert((head_key, ingress_id), submission_id); self.witnessed_submissions .insert(submission_id, record.clone()); + self.pending_witnessed_submission_ids.insert(submission_id); Ok(record) } @@ -3025,6 +3067,9 @@ impl WorldlineRuntime { .receipt_correlations_by_current_basis .get(¤t_basis) .cloned(), + previous_pending_submission: self + .pending_witnessed_submission_ids + .contains(&ticketed_ingress.submission_id), }); self.receipt_correlations_by_ticketed_ingress .insert(ticketed_ingress_id, record); @@ -3038,6 +3083,8 @@ impl WorldlineRuntime { .entry(current_basis) .or_default() .insert(ticketed_ingress_id); + self.pending_witnessed_submission_ids + .remove(&ticketed_ingress.submission_id); } Ok(()) } diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index c2f93242..6fe6997b 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -1575,22 +1575,14 @@ impl TrustedRuntimeHost { ) -> Result, TrustedRuntimeHostError> { let pending = self .runtime - .witnessed_submissions() + .pending_witnessed_submissions() .filter_map(|submission| { - matches!( - self.runtime - .observe_intent_outcome(&submission.submission_id), - IntentOutcomeObservation::Pending { .. } - ) - .then(|| { - self.runtime - .witnessed_submission_envelope(&submission.submission_id) - .and_then(|envelope| { - echo_operation_action_invocation_bytes_v1(envelope) - .map(|bytes| (submission.clone(), envelope.clone(), bytes.to_vec())) - }) - }) - .flatten() + self.runtime + .witnessed_submission_envelope(&submission.submission_id) + .and_then(|envelope| { + echo_operation_action_invocation_bytes_v1(envelope) + .map(|bytes| (submission.clone(), envelope.clone(), bytes.to_vec())) + }) }) .collect::>(); if pending.is_empty() { @@ -1650,16 +1642,18 @@ impl TrustedRuntimeHost { &admitted_actions, &self.echo_operation_evaluation_authority, )?; + let new_correlations = self + .runtime + .receipt_correlations() + .filter(|correlation| !existing_correlations.contains(&correlation.ticketed_ingress_id)) + .cloned() + .collect::>(); + let action_submission_by_ingress = new_correlations + .iter() + .map(|correlation| (correlation.ingress_id, correlation.submission_id)) + .collect::>(); let mut tick_wal_records = Vec::new(); if self.runtime_wal.is_some() { - let new_correlations = self - .runtime - .receipt_correlations() - .filter(|correlation| { - !existing_correlations.contains(&correlation.ticketed_ingress_id) - }) - .cloned() - .collect::>(); for correlation in new_correlations { let decision = match wal_tick_decision_from_observation( self.runtime @@ -1757,13 +1751,9 @@ impl TrustedRuntimeHost { } } for (ingress_id, outcome) in action_outcomes { - if let Some(submission) = self - .runtime - .witnessed_submissions() - .find(|submission| submission.ingress_id == ingress_id) - { + if let Some(submission_id) = action_submission_by_ingress.get(&ingress_id) { self.echo_operation_action_outcomes - .insert(submission.submission_id, outcome); + .insert(*submission_id, outcome); } } Ok(records) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index b97c7243..bc8671dc 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2738,6 +2738,7 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { .submit_intent_with_runtime_wal_ack(second_envelope) .expect("the second Action is durable before acknowledgement") .submission_id; + assert_eq!(host.runtime().pending_witnessed_submission_count(), 2); assert_eq!( host.runtime() @@ -2757,6 +2758,7 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { assert_eq!(steps.len(), 1); assert_eq!(steps[0].admitted_count, 2); assert_eq!(steps[0].worldline_tick_after.as_u64(), 1); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 0); for submission_id in [first_submission_id, second_submission_id] { let Some(EchoOperationActionOutcomeV1::Committed(receipt)) = @@ -2861,6 +2863,7 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { .engine() .installed_echo_operation_package_v1(package_id) .is_some()); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 0); for submission_id in [first_submission_id, second_submission_id] { assert!(matches!( recovered.echo_operation_action_outcome_v1(&submission_id), @@ -2931,6 +2934,7 @@ fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { .. } )); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); assert_eq!( host.runtime() .worldlines() @@ -2958,6 +2962,7 @@ fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { .. } )); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 1); recovered.install_echo_operation_action_admission_policy_v1(invocation_policy()); let steps = recovered .tick_once() @@ -2967,6 +2972,7 @@ fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { recovered.echo_operation_action_outcome_v1(&submission_id), Some(EchoOperationActionOutcomeV1::Committed(_)) )); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 0); let state = recovered .runtime() .worldlines() From ec0b1c08ae5311b6c9fd0f15e77ed910b0f01705 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:36:44 -0700 Subject: [PATCH 08/64] Fix: bound retained Action blocker allocation --- crates/warp-core/src/echo_operation.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index a8151237..9c504b16 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -2842,6 +2842,12 @@ pub(crate) fn recover_action_outcome_v1( let preparation_id = PreparedEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); let blocker_count = read_action_outcome_u64(bytes, &mut offset)?; + let maximum_encoded_blockers = + u64::try_from(bytes.len().saturating_sub(offset) / core::mem::size_of::()) + .unwrap_or(u64::MAX); + if blocker_count > maximum_encoded_blockers { + return Err(invalid_structure("Action blocker bytes are truncated")); + } let blocker_count = usize::try_from(blocker_count) .map_err(|_| invalid_structure("Action blocker count is not representable"))?; let mut blocked_by = Vec::with_capacity(blocker_count); @@ -5833,6 +5839,26 @@ mod tests { encode_canonical_cbor_v1(&CanonicalValueV1::Map(fields)).expect("fixture map encodes") } + #[test] + fn retained_action_outcome_rejects_impossible_blocker_count_before_allocation() { + let mut bytes = Vec::new(); + bytes.extend_from_slice(ACTION_OUTCOME_RECORD_MAGIC); + bytes.extend_from_slice(&digest(1)); + bytes.extend_from_slice(&digest(2)); + bytes.push(3); + bytes.extend_from_slice(&digest(3)); + bytes.extend_from_slice(&digest(4)); + bytes.extend_from_slice(&digest(5)); + bytes.extend_from_slice(&u64::MAX.to_le_bytes()); + + let error = recover_action_outcome_v1(&bytes) + .expect_err("an impossible blocker count must fail before allocation"); + assert_eq!( + error.kind(), + EchoOperationArtifactErrorKindV1::InvalidStructure + ); + } + #[test] fn retained_installation_rejects_identity_substitution() { let installed = retained_fixture_installation(); From 6968cbbaf7af49d5fd7ea82f64471f7ecff9b315 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:38:38 -0700 Subject: [PATCH 09/64] Fix: bind recovered Action outcomes to installed meaning --- crates/warp-core/src/trusted_runtime_host.rs | 17 ++++++++++++++--- .../executable_operation_pipeline_tests.rs | 6 ++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 6fe6997b..737b9cbf 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -465,6 +465,7 @@ impl TrustedRuntimeWalRecovery { &self.witnessed_submissions, &self.receipt_correlations, &self.provenance_entries, + &self.installed_echo_operations, &self.echo_operation_action_outcomes, ) } @@ -2172,6 +2173,7 @@ impl TrustedRuntimeWal { &witnessed_submissions, &receipt_correlations, &provenance_entries, + &installed_echo_operations, &echo_operation_action_outcomes, )?; let certificate = runtime_wal_recovery_certificate( @@ -4032,6 +4034,7 @@ fn validate_recovered_echo_operation_action_outcomes( witnessed_submissions: &WitnessedSubmissionPersistenceSnapshot, receipt_correlations: &[ReceiptCorrelationPersistenceRecord], provenance_entries: &[ProvenanceEntry], + installed_echo_operations: &[InstalledEchoOperationV1], outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], ) -> Result<(), TrustedRuntimeWalError> { let envelopes = witnessed_submissions @@ -4067,12 +4070,17 @@ fn validate_recovered_echo_operation_action_outcomes( .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; let invocation = inspect_action_invocation_v1(invocation_bytes) .map_err(|_| TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let installed = installed_echo_operations + .iter() + .find(|installed| installed.package_id() == invocation.package_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; if envelope.ingress_id() != *ingress_id { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } match outcome { EchoOperationActionOutcomeV1::Committed(receipt) => { - if receipt.tick_receipt_digest() != correlation.tick_receipt_digest + if validate_receipt_installation_v1(receipt, installed).is_err() + || receipt.tick_receipt_digest() != correlation.tick_receipt_digest || receipt.commit_id() != correlation.commit_hash || receipt.commit_global_tick() != Some(correlation.commit_global_tick) || receipt.worldline_tick_after() != correlation.worldline_tick_after @@ -4087,7 +4095,8 @@ fn validate_recovered_echo_operation_action_outcomes( } } EchoOperationActionOutcomeV1::Obstructed(obstruction) => { - if obstruction.package_id() != invocation.package_id + if obstruction.installed_operation_id() != installed.installed_operation_id() + || obstruction.package_id() != invocation.package_id || obstruction.invocation_id() != invocation.invocation_id || obstruction.evaluation_basis_id() != invocation.evaluation_basis.identity() { @@ -4095,11 +4104,13 @@ fn validate_recovered_echo_operation_action_outcomes( } } EchoOperationActionOutcomeV1::RejectedFootprintConflict { + installed_operation_id, invocation_id, blocked_by, .. } => { - if blocked_by.is_empty() + if *installed_operation_id != installed.installed_operation_id() + || blocked_by.is_empty() || blocked_by.windows(2).any(|pair| pair[0] >= pair[1]) || *invocation_id != invocation.invocation_id { diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index bc8671dc..8b8a1f3c 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2836,6 +2836,12 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { let mut adversarial = runtime_wal .recover_read_only() .expect("the honest composite Tick recovers"); + let mut missing_installation = adversarial.clone(); + missing_installation.installed_echo_operations.clear(); + assert!(matches!( + missing_installation.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); assert!( adversarial .echo_operation_action_outcomes From 4e6268c057cdc130d4d6fba99358303356e818cf Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:40:13 -0700 Subject: [PATCH 10/64] Fix: bind Action receipt entries to canonical ingress order --- crates/warp-core/src/coordinator.rs | 7 +++++++ crates/warp-core/src/echo_operation.rs | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 5bb29d9b..2488d3f2 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -1783,6 +1783,13 @@ impl WorldlineRuntime { }) .collect::>(); same_tick.sort_by_key(|candidate| candidate.ingress_id); + if same_tick.len() != receipt.entries().len() + || same_tick + .windows(2) + .any(|pair| pair[0].ingress_id >= pair[1].ingress_id) + { + return no_match(); + } let Some(index) = same_tick.iter().position(|candidate| { candidate.ticketed_ingress_id == correlation.ticketed_ingress_id }) else { diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 9c504b16..05fab856 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -4123,6 +4123,9 @@ pub enum EchoOperationCommitErrorV1 { /// The candidate count cannot be represented by Tick blocker indices. #[error("executable-operation Action batch has too many candidates")] TooManyCandidates, + /// More than one candidate claimed the same canonical ingress identity. + #[error("executable-operation Action batch contains duplicate ingress identity")] + DuplicateCandidateIngress, /// Test-only scheduler fault injected before Action Tick construction. #[cfg(all( feature = "native_rule_bootstrap", @@ -4177,11 +4180,21 @@ struct EchoOperationTerminalMaterialV1 { } pub(crate) fn commit_scheduler_action_batch_to_state_v1( - candidates: Vec, + mut candidates: Vec, state: &mut WorldlineState, commit_global_tick: GlobalTick, policy_id: u32, ) -> Result { + // This order is part of the receipt-correlation contract: coordinator + // outcome lookup maps the same Tick's ingress-sorted correlations onto + // these receipt entries by position. + candidates.sort_by_key(|candidate| candidate.ingress_id); + if candidates + .windows(2) + .any(|pair| pair[0].ingress_id == pair[1].ingress_id) + { + return Err(EchoOperationCommitErrorV1::DuplicateCandidateIngress); + } let state_root_before = state.state_root(); let tx_raw = state .current_tick() From 247550f2ceef08be33ba6154945e063ed00406f6 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:43:27 -0700 Subject: [PATCH 11/64] Fix: recompute composite Action receipt evidence --- crates/warp-core/src/echo_operation.rs | 74 +++++++++++++++++-- crates/warp-core/src/trusted_runtime_host.rs | 38 +++++++--- .../executable_operation_pipeline_tests.rs | 13 ++++ 3 files changed, 109 insertions(+), 16 deletions(-) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 05fab856..800d0173 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -3516,6 +3516,12 @@ impl EchoOperationReceiptV1 { self.composition_digest } + /// Replaces composition evidence for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_composition_digest_for_test(&mut self, composition_digest: Hash) { + self.composition_digest = Some(composition_digest); + } + /// Returns the closed terminal-outcome identity. #[must_use] pub const fn terminal_outcome_digest(&self) -> Hash { @@ -4426,17 +4432,71 @@ fn action_batch_rewrites_digest(prepared: &[&PreparedEchoOperationV1]) -> Hash { fn action_batch_composition_digest( prepared: &[&PreparedEchoOperationV1], committed_patch_digest: Hash, +) -> Hash { + action_batch_composition_digest_from_parts( + committed_patch_digest, + prepared.len(), + prepared.iter().map(|member| { + ( + member.preparation_id(), + member.patch().digest(), + member.result_id(), + member.evaluation_basis().identity(), + member.actual_footprint_digest(), + ) + }), + ) +} + +pub(crate) fn action_batch_composition_digest_from_receipts_v1( + receipts: &[&EchoOperationReceiptV1], + committed_patch_digest: Hash, +) -> Hash { + action_batch_composition_digest_from_parts( + committed_patch_digest, + receipts.len(), + receipts.iter().map(|receipt| { + ( + receipt.preparation_id, + receipt.prepared_patch_digest, + receipt.prepared_result_id, + receipt.evaluation_basis_id, + receipt.actual_footprint_digest, + ) + }), + ) +} + +fn action_batch_composition_digest_from_parts( + committed_patch_digest: Hash, + member_count: usize, + members: impl IntoIterator< + Item = ( + PreparedEchoOperationIdV1, + Hash, + EchoOperationResultIdV1, + EchoOperationEvaluationBasisIdV1, + Hash, + ), + >, ) -> Hash { let mut hasher = Hasher::new(); hasher.update(ACTION_BATCH_COMPOSITION_DOMAIN); hasher.update(&committed_patch_digest); - hasher.update(&(prepared.len() as u64).to_le_bytes()); - for member in prepared { - hasher.update(&member.preparation_id().as_hash()); - hasher.update(&member.patch().digest()); - hasher.update(&member.result_id().as_hash()); - hasher.update(&member.evaluation_basis().identity().as_hash()); - hasher.update(&member.actual_footprint_digest()); + hasher.update(&(member_count as u64).to_le_bytes()); + for ( + preparation_id, + prepared_patch_digest, + prepared_result_id, + evaluation_basis_id, + actual_footprint_digest, + ) in members + { + hasher.update(&preparation_id.as_hash()); + hasher.update(&prepared_patch_digest); + hasher.update(&prepared_result_id.as_hash()); + hasher.update(&evaluation_basis_id.as_hash()); + hasher.update(&actual_footprint_digest); } hasher.finalize().into() } diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 737b9cbf..eb6759b8 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -41,15 +41,15 @@ use crate::{ }, contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ - admit_action_invocation_v1, admit_invocation_v1, admit_package_v1, - commit_prepared_to_state, decode_invocation_route_v1, - echo_operation_action_invocation_bytes_v1, inspect_action_invocation_v1, - install_recovered_v1, installed_from_admitted, not_committed_basis_changed, - not_committed_evaluation_authority_mismatch, not_committed_installation_unavailable, - operation_descent_stack, prepare_operation_v1, recover_action_outcome_v1, - recover_committed_execution_receipt_v1, recover_installation_v1, retain_action_outcome_v1, - retain_committed_execution_v1, retain_installation_v1, validate_receipt_installation_v1, - EchoOperationEvaluationAuthorityV1, + action_batch_composition_digest_from_receipts_v1, admit_action_invocation_v1, + admit_invocation_v1, admit_package_v1, commit_prepared_to_state, + decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, + inspect_action_invocation_v1, install_recovered_v1, installed_from_admitted, + not_committed_basis_changed, not_committed_evaluation_authority_mismatch, + not_committed_installation_unavailable, operation_descent_stack, prepare_operation_v1, + recover_action_outcome_v1, recover_committed_execution_receipt_v1, recover_installation_v1, + retain_action_outcome_v1, retain_committed_execution_v1, retain_installation_v1, + validate_receipt_installation_v1, EchoOperationEvaluationAuthorityV1, }, provider_contract::admit_provider_contract_package_v1, AdmittedEchoOperationInvocationV1, AdmittedExecutableOperationPackageV1, @@ -4135,6 +4135,26 @@ fn validate_recovered_echo_operation_action_outcomes( if group.len() != tick_receipt.entries().len() { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } + let committed_receipts = group + .iter() + .filter_map(|(_, outcome, _, _)| match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => Some(receipt.as_ref()), + EchoOperationActionOutcomeV1::Obstructed(_) + | EchoOperationActionOutcomeV1::RejectedFootprintConflict { .. } => None, + }) + .collect::>(); + if !committed_receipts.is_empty() { + let expected_composition_digest = action_batch_composition_digest_from_receipts_v1( + &committed_receipts, + entry.expected.patch_digest, + ); + if committed_receipts + .iter() + .any(|receipt| receipt.composition_digest() != Some(expected_composition_digest)) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } for (index, (_, outcome, correlation, invocation)) in group.into_iter().enumerate() { if entry.commit_global_tick != correlation.commit_global_tick || entry.head_key != Some(correlation.head_key) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 8b8a1f3c..c94d8992 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2842,6 +2842,19 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { missing_installation.validate_echo_operation_action_outcomes_for_test(), Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) )); + let mut forged_composition = adversarial.clone(); + let Some((_, _, EchoOperationActionOutcomeV1::Committed(receipt))) = forged_composition + .echo_operation_action_outcomes + .iter_mut() + .find(|(_, _, outcome)| matches!(outcome, EchoOperationActionOutcomeV1::Committed(_))) + else { + panic!("the recovered composite Tick retains one committed Action receipt"); + }; + receipt.replace_composition_digest_for_test(digest("forged-composition")); + assert!(matches!( + forged_composition.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); assert!( adversarial .echo_operation_action_outcomes From e68ba0e15177a44c7efaaf9588ac5ba77ca99b9f Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:49:49 -0700 Subject: [PATCH 12/64] Fix: retain one decision record per scheduler Tick --- CHANGELOG.md | 9 +- crates/warp-core/src/causal_wal.rs | 142 ++++++++++++++++-- crates/warp-core/src/trusted_runtime_host.rs | 135 ++++++++++++----- .../executable_operation_pipeline_tests.rs | 3 +- 4 files changed, 233 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 956adb26..633604b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,11 @@ contribute to one composite Tick, while retaining candidate-specific application-basis propositions and per-Action typed outcomes. Footprint conflicts name earlier applied members; evaluator obstructions contribute no - operations. One scheduler WAL transaction retains each Action's receipt - correlation and typed outcome in canonical order followed by exactly one - replayable state delta. The decided Tick is durable before state, frontier, - receipt, or outcome publication, and fresh-host recovery validates every + operations. One scheduler WAL transaction retains exactly one batched Tick + decision record, then each Action's receipt correlation and typed outcome in + canonical order, followed by exactly one replayable state delta. The decided + Tick is durable before state, frontier, receipt, or outcome publication, and + fresh-host recovery validates every outcome against its exact envelope, invocation, installed operation, Tick entry, composite consequence, and state root. Direct operation prepare/commit remains a hidden transitional compatibility/test seam rather diff --git a/crates/warp-core/src/causal_wal.rs b/crates/warp-core/src/causal_wal.rs index f007b49d..e8b57349 100644 --- a/crates/warp-core/src/causal_wal.rs +++ b/crates/warp-core/src/causal_wal.rs @@ -73,6 +73,7 @@ use crate::{CausalTickReceiptRef, CAUSAL_TICK_RECEIPT_REF_LEN}; const WAL_FRAME_DOMAIN: &[u8] = b"echo:causal_wal:frame:v1\0"; const WAL_PAYLOAD_DOMAIN: &[u8] = b"echo:causal_wal:payload:v1\0"; const WAL_TICK_RECEIPT_MAGIC_V2: &[u8; 8] = b"ETICK002"; +const WAL_TICK_RECEIPT_BATCH_MAGIC_V3: &[u8; 8] = b"ETICK003"; const WAL_RECEIPT_CORRELATION_MAGIC_V2: &[u8; 8] = b"ERCOR002"; const LEGACY_TICK_RECEIPT_PAYLOAD_LEN: usize = 3 * core::mem::size_of::() + 1; const LEGACY_RECEIPT_CORRELATION_PREFIX_LEN: usize = 3 * core::mem::size_of::(); @@ -2502,6 +2503,108 @@ impl TickReceiptRecord { } } +#[derive(Clone, Debug, PartialEq, Eq)] +struct TickReceiptBatchRecord { + members: Vec, +} + +impl TickReceiptBatchRecord { + fn new(members: Vec) -> Result { + let Some(first) = members.first() else { + return Err(WalBuildError::TickBatchReceiptShapeMismatch); + }; + if members + .iter() + .any(|member| !same_scheduler_tick(&first.receipt_ref, &member.receipt_ref)) + { + return Err(WalBuildError::TickBatchReceiptShapeMismatch); + } + Ok(Self { members }) + } + + fn to_payload_bytes(&self) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(WAL_TICK_RECEIPT_BATCH_MAGIC_V3); + out.extend_from_slice(&len_u64(self.members.len()).to_le_bytes()); + for member in &self.members { + out.extend_from_slice(&member.receipt_ref.to_canonical_bytes()); + out.push(member.decision.code()); + } + out + } + + fn from_payload_bytes(bytes: &[u8]) -> Result { + let mut cursor = WalPayloadCursor::new(bytes); + if cursor.read_exact(WAL_TICK_RECEIPT_BATCH_MAGIC_V3.len())? + != WAL_TICK_RECEIPT_BATCH_MAGIC_V3 + { + return Err(WalDecodeError::InvalidRecordMagic { + record_kind: "tick-receipt-batch", + }); + } + let member_count = cursor.read_u64()?; + let member_width = CAUSAL_TICK_RECEIPT_REF_LEN + 1; + let maximum_encoded_members = u64::try_from( + bytes + .len() + .saturating_sub(WAL_TICK_RECEIPT_BATCH_MAGIC_V3.len() + 8) + / member_width, + ) + .unwrap_or(u64::MAX); + if member_count == 0 || member_count > maximum_encoded_members { + return Err(WalDecodeError::InvalidEmbeddedFrame); + } + let member_count = + usize::try_from(member_count).map_err(|_| WalDecodeError::UnexpectedEof)?; + let mut members = Vec::with_capacity(member_count); + for _ in 0..member_count { + let receipt_ref = CausalTickReceiptRef::from_canonical_bytes( + cursor + .read_exact(CAUSAL_TICK_RECEIPT_REF_LEN)? + .try_into() + .map_err(|_| WalDecodeError::UnexpectedEof)?, + ); + let decision = WalTickDecision::from_code(cursor.read_u8()?)?; + members.push(TickReceiptRecord { + receipt_ref, + decision, + }); + } + cursor.finish()?; + let first = &members[0].receipt_ref; + if members + .iter() + .any(|member| !same_scheduler_tick(first, &member.receipt_ref)) + { + return Err(WalDecodeError::InvalidEmbeddedFrame); + } + Ok(Self { members }) + } +} + +pub(crate) fn decode_tick_receipt_records( + bytes: &[u8], +) -> Result, WalDecodeError> { + if bytes.starts_with(WAL_TICK_RECEIPT_BATCH_MAGIC_V3) { + TickReceiptBatchRecord::from_payload_bytes(bytes).map(|batch| batch.members) + } else { + TickReceiptRecord::from_payload_bytes(bytes).map(|receipt| vec![receipt]) + } +} + +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +pub(crate) fn tick_receipt_payload_is_batch(bytes: &[u8]) -> bool { + bytes.starts_with(WAL_TICK_RECEIPT_BATCH_MAGIC_V3) +} + +fn same_scheduler_tick(left: &CausalTickReceiptRef, right: &CausalTickReceiptRef) -> bool { + left.worldline_id == right.worldline_id + && left.worldline_tick_after == right.worldline_tick_after + && left.commit_global_tick == right.commit_global_tick + && left.commit_hash == right.commit_hash + && left.receipt_content_digest == right.receipt_content_digest +} + /// WAL receipt correlation record payload. #[derive(Clone, Debug, PartialEq, Eq)] pub struct WalReceiptCorrelationRecord { @@ -7823,14 +7926,32 @@ pub(crate) fn build_replayable_tick_batch_transaction( { return Err(WalBuildError::RuntimeStateDeltaReceiptMismatch); } - for (receipt, correlation, action_outcome) in receipts { - push_tick_receipt_records(&mut builder, receipt, &correlation)?; - if let Some(action_outcome) = action_outcome { + if action_outcome_count == receipts.len() { + let receipt_batch = + TickReceiptBatchRecord::new(receipts.iter().map(|(receipt, _, _)| *receipt).collect())?; + builder.push_record( + WalRecordKind::TickReceiptRecorded, + receipt_batch.to_payload_bytes(), + )?; + for (receipt, correlation, action_outcome) in receipts { + if receipt.receipt_ref != correlation.receipt_ref { + return Err(WalBuildError::ReceiptCorrelationMismatch); + } + builder.push_record( + WalRecordKind::ReceiptCorrelationRecorded, + correlation.to_payload_bytes(), + )?; + let action_outcome = + action_outcome.ok_or(WalBuildError::TickBatchActionOutcomeShapeMismatch)?; builder.push_record( WalRecordKind::ExecutableOperationActionOutcomeRecorded, action_outcome, )?; } + } else { + for (receipt, correlation, _) in receipts { + push_tick_receipt_records(&mut builder, receipt, &correlation)?; + } } builder.push_record( WalRecordKind::RuntimeStateDeltaRecorded, @@ -8450,9 +8571,9 @@ pub fn recover_submission_index( index.insert_acceptance_record(record)?; } WalRecordKind::TickReceiptRecorded => { - let receipt = - TickReceiptRecord::from_payload_bytes(&frame.payload.canonical_bytes)?; - index.apply_tick_receipt_record(receipt)?; + for receipt in decode_tick_receipt_records(&frame.payload.canonical_bytes)? { + index.apply_tick_receipt_record(receipt)?; + } } _ => {} } @@ -8470,9 +8591,9 @@ pub fn recover_receipt_index( for frame in &transaction.frames { match frame.header.record_kind { WalRecordKind::TickReceiptRecorded => { - let receipt = - TickReceiptRecord::from_payload_bytes(&frame.payload.canonical_bytes)?; - index.apply_tick_receipt_record(receipt)?; + for receipt in decode_tick_receipt_records(&frame.payload.canonical_bytes)? { + index.apply_tick_receipt_record(receipt)?; + } } WalRecordKind::ReceiptCorrelationRecorded => { let correlation = WalReceiptCorrelationRecord::from_payload_bytes( @@ -8904,6 +9025,9 @@ pub enum WalBuildError { /// A scheduler Tick batch mixes Action and non-Action receipt members. #[error("WAL scheduler Tick batch has inconsistent Action outcome framing")] TickBatchActionOutcomeShapeMismatch, + /// A scheduler Tick batch contains no receipt members or spans Ticks. + #[error("WAL scheduler Tick batch receipt shape is inconsistent")] + TickBatchReceiptShapeMismatch, /// Validation failed. #[error(transparent)] Validation(#[from] WalValidationError), diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index eb6759b8..0b9899b3 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -24,9 +24,10 @@ use crate::{ build_replayable_tick_batch_transaction, build_replayable_tick_transaction, build_submission_acceptance_with_material_transaction, causal_anchor_frontier_digest_from_evidence, causal_anchor_genesis_frontier_digest, - causal_history_genesis_frontier_digest, logical_causal_history_frontier_digest, - recover_filesystem_store, recover_from_frames_and_commits, recover_receipt_index, - recover_submission_index, recovered_submission_receipt_index_root, + causal_history_genesis_frontier_digest, decode_tick_receipt_records, + logical_causal_history_frontier_digest, recover_filesystem_store, + recover_from_frames_and_commits, recover_receipt_index, recover_submission_index, + recovered_submission_receipt_index_root, tick_receipt_payload_is_batch, trusted_runtime_wal_digest, validate_recovered_causal_anchor_history, AffectedFrontier, AffectedFrontierKind, FilesystemWalStore, InMemoryWalStore, Lsn, PayloadCodecId, PayloadSchemaId, RecoveredCausalAnchorAdmission, RecoveredReceiptIndex, @@ -4314,53 +4315,105 @@ fn tick_record_batch_from_transaction( )); } let mut records = Vec::new(); - let mut index = 0; - while index < state_delta_index { - let Some(receipt_frame) = transaction.frames.get(index) else { - return Err(decode_trusted_runtime_wal_payload( - WalDecodeError::InvalidEmbeddedFrame, - )); - }; - let Some(correlation_frame) = transaction.frames.get(index + 1) else { - return Err(decode_trusted_runtime_wal_payload( - WalDecodeError::InvalidEmbeddedFrame, - )); - }; - if receipt_frame.header.record_kind != WalRecordKind::TickReceiptRecorded - || correlation_frame.header.record_kind != WalRecordKind::ReceiptCorrelationRecorded - { - return Err(decode_trusted_runtime_wal_payload( - WalDecodeError::InvalidEmbeddedFrame, - )); - } - let receipt = TickReceiptRecord::from_payload_bytes(&receipt_frame.payload.canonical_bytes) + let first_frame = &transaction.frames[0]; + if first_frame.header.record_kind != WalRecordKind::TickReceiptRecorded { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + if tick_receipt_payload_is_batch(&first_frame.payload.canonical_bytes) { + let receipts = decode_tick_receipt_records(&first_frame.payload.canonical_bytes) .map_err(decode_trusted_runtime_wal_payload)?; - let correlation = WalReceiptCorrelationRecord::from_payload_bytes( - &correlation_frame.payload.canonical_bytes, - ) - .map_err(decode_trusted_runtime_wal_payload)?; - if correlation.receipt_ref != receipt.receipt_ref { + let mut index = 1; + for receipt in receipts { + let Some(correlation_frame) = transaction.frames.get(index) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + let Some(outcome_frame) = transaction.frames.get(index + 1) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + if correlation_frame.header.record_kind != WalRecordKind::ReceiptCorrelationRecorded + || outcome_frame.header.record_kind + != WalRecordKind::ExecutableOperationActionOutcomeRecorded + { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let correlation = WalReceiptCorrelationRecord::from_payload_bytes( + &correlation_frame.payload.canonical_bytes, + ) + .map_err(decode_trusted_runtime_wal_payload)?; + let action_outcome = recover_action_outcome_v1(&outcome_frame.payload.canonical_bytes)?; + if correlation.receipt_ref != receipt.receipt_ref + || action_outcome.0 != receipt.receipt_ref.submission_id + { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + records.push((receipt, correlation, Some(action_outcome))); + index += 2; + } + if index != state_delta_index { return Err(decode_trusted_runtime_wal_payload( WalDecodeError::InvalidEmbeddedFrame, )); } - index += 2; - let action_outcome = if transaction.frames.get(index).is_some_and(|frame| { - frame.header.record_kind == WalRecordKind::ExecutableOperationActionOutcomeRecorded - }) { - let frame = &transaction.frames[index]; - index += 1; - let recovered = recover_action_outcome_v1(&frame.payload.canonical_bytes)?; - if recovered.0 != receipt.receipt_ref.submission_id { + } else { + let mut index = 0; + while index < state_delta_index { + let Some(receipt_frame) = transaction.frames.get(index) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + let Some(correlation_frame) = transaction.frames.get(index + 1) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + if receipt_frame.header.record_kind != WalRecordKind::TickReceiptRecorded + || correlation_frame.header.record_kind != WalRecordKind::ReceiptCorrelationRecorded + { return Err(decode_trusted_runtime_wal_payload( WalDecodeError::InvalidEmbeddedFrame, )); } - Some(recovered) - } else { - None - }; - records.push((receipt, correlation, action_outcome)); + let receipt = + TickReceiptRecord::from_payload_bytes(&receipt_frame.payload.canonical_bytes) + .map_err(decode_trusted_runtime_wal_payload)?; + let correlation = WalReceiptCorrelationRecord::from_payload_bytes( + &correlation_frame.payload.canonical_bytes, + ) + .map_err(decode_trusted_runtime_wal_payload)?; + if correlation.receipt_ref != receipt.receipt_ref { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + index += 2; + let action_outcome = if transaction.frames.get(index).is_some_and(|frame| { + frame.header.record_kind == WalRecordKind::ExecutableOperationActionOutcomeRecorded + }) { + let frame = &transaction.frames[index]; + index += 1; + let recovered = recover_action_outcome_v1(&frame.payload.canonical_bytes)?; + if recovered.0 != receipt.receipt_ref.submission_id { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + Some(recovered) + } else { + None + }; + records.push((receipt, correlation, action_outcome)); + } } let action_outcome_count = records .iter() diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index c94d8992..895e3891 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2825,12 +2825,11 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { WalRecordKind::TickReceiptRecorded, WalRecordKind::ReceiptCorrelationRecorded, WalRecordKind::ExecutableOperationActionOutcomeRecorded, - WalRecordKind::TickReceiptRecorded, WalRecordKind::ReceiptCorrelationRecorded, WalRecordKind::ExecutableOperationActionOutcomeRecorded, WalRecordKind::RuntimeStateDeltaRecorded, ], - "one composite Tick retains per-Action decisions and one state delta" + "one composite Tick retains one decision record, per-Action outcomes, and one state delta" ); let mut adversarial = runtime_wal From ea978cd39920440d5e8d32e3ab70dde828037bae Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 05:52:04 -0700 Subject: [PATCH 13/64] Docs: align scheduler Action composition claims --- README.md | 33 ++++++++++--------- ...uler-owned-executable-operation-actions.md | 12 ++++--- docs/topics/GeneratedRules.md | 21 +++++++----- docs/topics/WAL.md | 11 ++++--- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 106167fa..49ab07be 100644 --- a/README.md +++ b/README.md @@ -175,25 +175,26 @@ host-constructed `InstalledContractPackage` values. Separately, Echo can now admit and install an exact canonical `ExecutableOperationPackageV1`, admit an exact-basis invocation under explicit authority and delegated budget, evaluate its data-only -`EchoOperationProgramV1` privately, and either commit one patch or return typed -noncommit evidence. Only committed operation consequences enter the -operation-tick WAL. The initial generic program performs an anchored typed-node -alpha-attachment compare-and-set. Its receipt binds the admitted package, -operation, subordinate program, invocation, complete evaluation basis, -authority, declared and actual footprints, budgets, patch, result, and terminal -outcome. Runtime-control installation and committed execution-kernel records -permit callback-free fresh-host recovery. Program bytes explicitly bind the -interpreter and intrinsic profiles, while the parent patch and singleton tick -evidence bind the admitted installation. A program digest cannot confer -operation identity, invocability, or authority. +`EchoOperationProgramV1` privately during scheduler-owned Tick construction, +and either commit its member consequence or return typed noncommit evidence. +Two independent, same-basis executable Actions can enter one atomic Tick. The +initial generic programs perform anchored typed-node alpha-attachment +compare-and-set and create-if-absent operations. Their receipts bind the +admitted package, operation, subordinate program, invocation, complete +evaluation basis, authority, declared and actual footprints, budgets, patch, +result, composition, and terminal outcome. Runtime-control installation and +scheduler Tick records permit callback-free fresh-host recovery. Program bytes +explicitly bind the interpreter and intrinsic profiles, while Tick evidence +binds each admitted installation. A program digest cannot confer operation +identity, invocability, or authority. That generic runtime witness is not yet the Jim/Jedit vertical. No real Edict compiler output, Jedit rope lawpack, or `ReplaceRange` operation uses it, and it -does not yet claim structurally separate target verification, scheduler batch -composition, or independently implemented semantic conformance. It also -temporarily reuses `TrustedRuntimeHost`'s joint `native_rule_bootstrap` and -`trusted_runtime` feature gate. The program itself has no native hooks, but the -host surface must be decoupled from the legacy bootstrap feature before a +does not yet claim structurally separate target verification, cross-category +scheduler composition, or independently implemented semantic conformance. It +also temporarily reuses `TrustedRuntimeHost`'s joint `native_rule_bootstrap` +and `trusted_runtime` feature gate. The program itself has no native hooks, but +the host surface must be decoupled from the legacy bootstrap feature before a product can remove that compatibility feature. The following sequence is the existing Wesley bootstrap fixture: diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md index 1d4b1856..bb1359e5 100644 --- a/docs/adr/0025-scheduler-owned-executable-operation-actions.md +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -132,14 +132,16 @@ semantics are not inferred from a generic receipt label. A scheduler Tick WAL transaction contains, in canonical Action order: ```text -TickReceiptRecorded + ReceiptCorrelationRecorded + ActionOutcomeRecorded -... repeated for each Action ... +TickReceiptRecorded +ReceiptCorrelationRecorded + ActionOutcomeRecorded +... one correlation/outcome pair per Action ... RuntimeStateDeltaRecorded ``` -Every receipt record cites the same Tick receipt content digest and the -submission-specific causal coordinate. The state delta appears exactly once -because the Tick has exactly one atomic state consequence. +The single batched Tick receipt record retains every per-Action scheduler +decision and exact causal receipt coordinate. Each following correlation and +typed outcome names its corresponding Action. The state delta appears exactly +once because the Tick has exactly one atomic state consequence. The WAL transaction is committed before the live runtime, frontier, provenance, Action outcomes, or receipts are published. On append failure, live state is diff --git a/docs/topics/GeneratedRules.md b/docs/topics/GeneratedRules.md index d8208a79..93ac29e4 100644 --- a/docs/topics/GeneratedRules.md +++ b/docs/topics/GeneratedRules.md @@ -178,15 +178,18 @@ invocability. The subordinate program supplies executable meaning only. Echo cannot install or invoke a naked program digest, and a generated artifact does not confer caller authority. Provider v1 remains stable while existing consumers migrate; it is not silently reinterpreted as this new category. -The program bytes bind their interpreter and intrinsic profiles directly. The -parent patch and singleton tick entry use the admitted installed-operation -identity as their rule-pack/rule identity, so two packages that reuse identical -program bytes do not collapse into one causal operation identity. -The current slice composes one preparation and does not claim general scheduler -batch integration, Graft-style multi-record mutation, or independent -implementation evidence. Its canonical decoding, identity recomputation, -budget and footprint checks, exact-basis comparison, deterministic -repeatability, and WAL recovery are deterministic self-validation. +The program bytes bind their interpreter and intrinsic profiles directly. +Direct compatibility commits retain their singleton evidence, while +scheduler-owned Action Ticks use the admitted installed-operation identity for +each receipt-entry rule identity and bind every applied preparation into one +canonical composite consequence. Two packages that reuse identical program +bytes therefore do not collapse into one causal operation identity. +The current slice composes multiple independent, same-basis executable Actions +inside one scheduler-owned Tick. It does not claim cross-category scheduler +composition, Graft-style multi-record mutation, or independent implementation +evidence. Its canonical decoding, identity recomputation, budget and footprint +checks, exact-basis comparison, deterministic repeatability, and WAL recovery +are deterministic self-validation. For descended targets, the retained footprint and patch inputs include every portal attachment in the validated root-to-target reachability chain. diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index 5ade6c24..1c28665c 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -91,11 +91,12 @@ same causal event and the retained state delta to bind that receipt's content commitment before the transaction can commit. Legacy singleton scheduler transactions contain exactly one receipt, correlation, and runtime state-delta frame. Scheduler-owned -executable-operation Action Ticks instead retain one ordered receipt, -correlation, and typed Action-outcome group per selected Action followed by -exactly one runtime state delta for the whole Tick. Recovery rejects missing, -duplicated, reordered, mixed, or cross-Tick claims rather than selecting -whichever frame appears first. +executable-operation Action Ticks instead retain exactly one batched Tick +receipt record containing every per-Action decision, one ordered correlation +and typed Action-outcome pair per selected Action, and exactly one runtime state +delta for the whole Tick. Recovery rejects missing, duplicated, reordered, +mixed, or cross-Tick claims rather than selecting whichever frame appears +first. WAL activation is non-lossy: if the live host contains submissions, staged ingress, receipt correlations, provenance, pending inbox work, cycle progress, or worldline state that recovered WAL evidence cannot reproduce, activation From aa4c1d8abbb346801c8b7314b8db48d494997606 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:30:50 -0700 Subject: [PATCH 14/64] Fix: quarantine unadmittable executable Actions --- crates/warp-core/src/trusted_runtime_host.rs | 87 ++++++++++---- .../executable_operation_pipeline_tests.rs | 111 ++++++++++++++++++ 2 files changed, 175 insertions(+), 23 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 0b9899b3..d970b66b 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -61,23 +61,23 @@ use crate::{ ContractOperationKind, EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorV1, EchoOperationAdmissionPolicyV1, EchoOperationApplicationBasisV1, EchoOperationArtifactErrorV1, EchoOperationCommitErrorV1, EchoOperationEvaluationBasisV1, EchoOperationExecutionEvidenceV1, - EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorV1, - EchoOperationInvocationAdmissionPolicyV1, EchoOperationPreparationV1, EchoOperationReceiptV1, - EchoOperationTerminalPostureV1, Engine, IngressCausalParent, IngressEnvelope, - IngressEnvelopeDecodeError, IngressPayload, IngressSubmissionGeneration, - InstalledContractPackage, InstalledContractPackageError, InstalledContractPackageRecord, - InstalledEchoOperationV1, InstalledProviderContractPackageRecordV1, IntentOutcome, - IntentOutcomeDecision, IntentOutcomeObservation, IntentSubmissionHandle, - IntentSubmissionRecord, ObservationArtifact, ObservationError, ObservationRequest, - ObservationService, OpticAdmissionTicket, PreparedEchoOperationV1, ProvenanceEntry, - ProvenanceService, ProvenanceStore, ProviderContractAdmissionError, - ProviderContractAdmissionPolicyV1, ProviderContractInstallationError, - ProviderContractPackageInstallerV1, ProviderContractPackageProposalV1, - ProviderPackageReferenceV1, ReceiptCorrelationPersistenceRecord, ReceiptCorrelationRecord, - RetainedProvenanceError, RuntimeError, SchedulerCoordinator, StepRecord, - TickReceiptDisposition, TickReceiptRejection, TicketedRuntimeIngressAuthority, - TicketedRuntimeIngressDisposition, WitnessedSubmissionPersistenceRecord, - WitnessedSubmissionPersistenceSnapshot, WorldlineRuntime, + EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorKindV1, + EchoOperationInvocationAdmissionErrorV1, EchoOperationInvocationAdmissionPolicyV1, + EchoOperationPreparationV1, EchoOperationReceiptV1, EchoOperationTerminalPostureV1, Engine, + IngressCausalParent, IngressEnvelope, IngressEnvelopeDecodeError, IngressPayload, + IngressSubmissionGeneration, InstalledContractPackage, InstalledContractPackageError, + InstalledContractPackageRecord, InstalledEchoOperationV1, + InstalledProviderContractPackageRecordV1, IntentOutcome, IntentOutcomeDecision, + IntentOutcomeObservation, IntentSubmissionHandle, IntentSubmissionRecord, ObservationArtifact, + ObservationError, ObservationRequest, ObservationService, OpticAdmissionTicket, + PreparedEchoOperationV1, ProvenanceEntry, ProvenanceService, ProvenanceStore, + ProviderContractAdmissionError, ProviderContractAdmissionPolicyV1, + ProviderContractInstallationError, ProviderContractPackageInstallerV1, + ProviderContractPackageProposalV1, ProviderPackageReferenceV1, + ReceiptCorrelationPersistenceRecord, ReceiptCorrelationRecord, RetainedProvenanceError, + RuntimeError, SchedulerCoordinator, StepRecord, TickReceiptDisposition, TickReceiptRejection, + TicketedRuntimeIngressAuthority, TicketedRuntimeIngressDisposition, + WitnessedSubmissionPersistenceRecord, WitnessedSubmissionPersistenceSnapshot, WorldlineRuntime, }; use crate::{Hash, HistoryError}; @@ -643,6 +643,8 @@ pub struct TrustedRuntimeHost { echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1, echo_operation_action_admission_policy: Option, echo_operation_action_outcomes: BTreeMap, + echo_operation_action_admission_obstructions: + BTreeMap, } impl TrustedRuntimeHost { @@ -663,6 +665,7 @@ impl TrustedRuntimeHost { echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), echo_operation_action_admission_policy: None, echo_operation_action_outcomes: BTreeMap::new(), + echo_operation_action_admission_obstructions: BTreeMap::new(), }) } @@ -682,6 +685,7 @@ impl TrustedRuntimeHost { echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), echo_operation_action_admission_policy: None, echo_operation_action_outcomes: BTreeMap::new(), + echo_operation_action_admission_obstructions: BTreeMap::new(), } } @@ -716,6 +720,7 @@ impl TrustedRuntimeHost { policy: EchoOperationInvocationAdmissionPolicyV1, ) { self.echo_operation_action_admission_policy = Some(policy); + self.echo_operation_action_admission_obstructions.clear(); } /// Returns the typed scheduler-owned outcome for one executable-operation @@ -728,6 +733,18 @@ impl TrustedRuntimeHost { self.echo_operation_action_outcomes.get(submission_id) } + /// Returns the typed pending posture for an executable-operation Action + /// which runtime-owned admission cannot currently authorize. + #[must_use] + pub fn echo_operation_action_admission_obstruction_v1( + &self, + submission_id: &Hash, + ) -> Option { + self.echo_operation_action_admission_obstructions + .get(submission_id) + .copied() + } + /// Admits exact executable-operation package bytes under independent policy. /// /// A successful result is an opaque admission token. It does not install @@ -773,6 +790,7 @@ impl TrustedRuntimeHost { } self.engine .restore_recovered_echo_operation_packages_v1(core::slice::from_ref(&installed))?; + self.echo_operation_action_admission_obstructions.clear(); Ok(installed) } @@ -1062,6 +1080,7 @@ impl TrustedRuntimeHost { .iter() .map(|(submission_id, _, outcome)| (*submission_id, outcome.clone())) .collect(); + self.echo_operation_action_admission_obstructions.clear(); self.runtime = restored_runtime; self.provenance = restored_provenance; self.runtime_wal = Some(runtime_wal); @@ -1590,18 +1609,38 @@ impl TrustedRuntimeHost { if pending.is_empty() { return Ok(BTreeMap::new()); } - let policy = self - .echo_operation_action_admission_policy - .ok_or(TrustedRuntimeHostError::EchoOperationActionAdmissionPolicyUnavailable)?; + let Some(policy) = self.echo_operation_action_admission_policy else { + return Ok(BTreeMap::new()); + }; let mut admitted_actions = BTreeMap::new(); for (submission, envelope, invocation_bytes) in pending { - let (package_id, _) = decode_invocation_route_v1(&invocation_bytes)?; - let admitted = admit_action_invocation_v1( + if self + .echo_operation_action_admission_obstructions + .contains_key(&submission.submission_id) + { + continue; + } + let package_id = match decode_invocation_route_v1(&invocation_bytes) { + Ok((package_id, _)) => package_id, + Err(error) => { + self.echo_operation_action_admission_obstructions + .insert(submission.submission_id, error.kind()); + continue; + } + }; + let admitted = match admit_action_invocation_v1( self.engine.installed_echo_operation_package_v1(package_id), policy, &invocation_bytes, self.echo_operation_evaluation_authority.clone(), - )?; + ) { + Ok(admitted) => admitted, + Err(error) => { + self.echo_operation_action_admission_obstructions + .insert(submission.submission_id, error.kind()); + continue; + } + }; let admission_digest = echo_operation_action_admission_digest(&submission, &admitted); self.runtime.ingest_echo_operation_action_v1( &TicketedRuntimeIngressAuthority::assume_runtime_owner(), @@ -1609,6 +1648,8 @@ impl TrustedRuntimeHost { admission_digest, envelope, )?; + self.echo_operation_action_admission_obstructions + .remove(&submission.submission_id); admitted_actions.insert(submission.ingress_id, admitted); } Ok(admitted_actions) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 895e3891..72a34517 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3117,6 +3117,117 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { ); } +#[test] +fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { + let wal_dir = TempWalDir::new(); + let unavailable_submission_id; + let valid_submission_id; + let head_key; + let node; + + { + let (mut host, fixture_head, fixture_node) = fixture_host(); + head_key = fixture_head; + node = fixture_node; + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the admission-obstruction fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let unavailable_package_bytes = operation_package_at( + "echo.fixture.Unavailable.v1", + make_type_id("operation-fixture-node"), + make_type_id("operation-fixture-atom"), + ) + .to_canonical_bytes() + .expect("the unavailable package is canonical"); + let unavailable_package_id = + warp_core::echo_operation_package_id_v1(&unavailable_package_bytes); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis()) + .expect("Echo resolves the current exact parent basis"); + let unavailable_invocation = + EchoOperationInvocationV1::anchored_node_attachment_compare_and_set( + unavailable_package_id, + "echo.fixture.Unavailable.v1", + evaluation_basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + node, + warp_core::echo_operation_atom_value_digest_v1( + make_type_id("operation-fixture-atom"), + b"before", + ), + b"must-not-appear".to_vec(), + ) + .to_canonical_bytes() + .expect("the unavailable-package invocation is canonical"); + unavailable_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + unavailable_invocation, + ) + .expect("the unavailable-package invocation becomes an Action"), + ) + .expect("the unavailable-package Action is durable before acknowledgement") + .submission_id; + + let valid_invocation = + action_invocation(&host, &installed, head_key, node, b"before", b"valid-after"); + valid_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + valid_invocation, + ) + .expect("the valid invocation becomes an Action"), + ) + .expect("the valid Action is durable before acknowledgement") + .submission_id; + + let steps = host + .tick_once() + .expect("one unadmittable Action cannot abort unrelated scheduler work"); + assert_eq!(steps.len(), 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + assert_eq!( + host.echo_operation_action_admission_obstruction_v1(&unavailable_submission_id), + Some(EchoOperationInvocationAdmissionErrorKindV1::OperationUnavailable) + ); + assert!(host + .echo_operation_action_outcome_v1(&unavailable_submission_id) + .is_none()); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); + } + + let (mut recovered, recovered_head, recovered_node) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("fresh-host recovery restores the valid Tick and pending Action"); + recovered.install_echo_operation_action_admission_policy_v1(invocation_policy()); + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + let steps = recovered + .tick_once() + .expect("the recovered unavailable-package Action is quarantined"); + assert!(steps.is_empty()); + assert_eq!( + recovered.echo_operation_action_admission_obstruction_v1(&unavailable_submission_id), + Some(EchoOperationInvocationAdmissionErrorKindV1::OperationUnavailable) + ); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 1); + assert_eq!(recovered_head, head_key); + assert_eq!(recovered_node, node); +} + #[test] fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { let wal_dir = TempWalDir::new(); From 3c11a641516a73289d21c3168d279c4773e9d4f9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:32:39 -0700 Subject: [PATCH 15/64] Fix: validate durable scheduler Action decisions --- crates/warp-core/src/trusted_runtime_host.rs | 40 +++++++++++++++++++ .../executable_operation_pipeline_tests.rs | 13 +++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index d970b66b..72b0ea14 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -452,6 +452,7 @@ pub struct TrustedRuntimeWalRecovery { /// Typed executable-operation Action outcomes reconstructed from /// scheduler-owned Tick records, keyed by witnessed submission. pub echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, + echo_operation_action_decisions: BTreeMap, causal_history_frontiers: Vec, } @@ -468,9 +469,21 @@ impl TrustedRuntimeWalRecovery { &self.provenance_entries, &self.installed_echo_operations, &self.echo_operation_action_outcomes, + &self.echo_operation_action_decisions, ) } + /// Replaces one recovered scheduler decision for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_decision_for_test( + &mut self, + submission_id: Hash, + decision: WalTickDecision, + ) { + self.echo_operation_action_decisions + .insert(submission_id, decision); + } + /// Recomputes the certificate's canonical index root from recovered evidence. /// /// # Errors @@ -2207,6 +2220,7 @@ impl TrustedRuntimeWal { let provenance_entries = runtime_state.provenance_entries; let receipt_correlations = runtime_state.receipt_correlations; let echo_operation_action_outcomes = runtime_state.echo_operation_action_outcomes; + let echo_operation_action_decisions = runtime_state.echo_operation_action_decisions; let missing_runtime_state_deltas = runtime_state.missing_runtime_state_deltas; let installed_echo_operations = operation_material.installations; let echo_operation_receipts = operation_material.receipts; @@ -2217,6 +2231,7 @@ impl TrustedRuntimeWal { &provenance_entries, &installed_echo_operations, &echo_operation_action_outcomes, + &echo_operation_action_decisions, )?; let certificate = runtime_wal_recovery_certificate( &report, @@ -2246,6 +2261,7 @@ impl TrustedRuntimeWal { installed_echo_operations, echo_operation_receipts, echo_operation_action_outcomes, + echo_operation_action_decisions, causal_history_frontiers, }) } @@ -2613,11 +2629,15 @@ impl TrustedRuntimeWal { let action_outcome = action_outcomes_by_ingress .get(&correlation.ingress_id) .map(|outcome| { + if wal_tick_decision_for_action_outcome(outcome) != *decision { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } retain_action_outcome_v1( correlation.submission_id, correlation.ingress_id, outcome, ) + .map_err(TrustedRuntimeWalError::from) }) .transpose()?; records.push((receipt, wal_correlation, action_outcome)); @@ -3887,6 +3907,7 @@ struct RecoveredRuntimeStateMaterial { provenance_entries: Vec, receipt_correlations: Vec, echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, + echo_operation_action_decisions: BTreeMap, missing_runtime_state_deltas: Vec, } @@ -3896,6 +3917,7 @@ fn recover_runtime_state_delta_material( let mut entries_by_coordinate = BTreeMap::new(); let mut correlations_by_submission = BTreeMap::new(); let mut action_outcomes_by_submission = BTreeMap::new(); + let mut action_decisions_by_submission = BTreeMap::new(); let mut submission_by_ticket = BTreeMap::new(); let mut missing = Vec::new(); for transaction in &report.transactions { @@ -3998,6 +4020,9 @@ fn recover_runtime_state_delta_material( || action_outcomes_by_submission .get(&submission_id) .is_some_and(|existing| existing != &(ingress_id, outcome.clone())) + || action_decisions_by_submission + .insert(submission_id, receipt.decision) + .is_some_and(|existing| existing != receipt.decision) { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } @@ -4043,6 +4068,7 @@ fn recover_runtime_state_delta_material( provenance_entries: entries, receipt_correlations: correlations, echo_operation_action_outcomes, + echo_operation_action_decisions: action_decisions_by_submission, missing_runtime_state_deltas: missing, }) } @@ -4078,6 +4104,7 @@ fn validate_recovered_echo_operation_action_outcomes( provenance_entries: &[ProvenanceEntry], installed_echo_operations: &[InstalledEchoOperationV1], outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], + decisions: &BTreeMap, ) -> Result<(), TrustedRuntimeWalError> { let envelopes = witnessed_submissions .records() @@ -4102,6 +4129,9 @@ fn validate_recovered_echo_operation_action_outcomes( )>, >::new(); for (submission_id, ingress_id, outcome) in outcomes { + if decisions.get(submission_id) != Some(&wal_tick_decision_for_action_outcome(outcome)) { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } let envelope = envelopes .get(submission_id) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; @@ -4257,6 +4287,16 @@ fn validate_recovered_echo_operation_action_outcomes( Ok(()) } +fn wal_tick_decision_for_action_outcome(outcome: &EchoOperationActionOutcomeV1) -> WalTickDecision { + match outcome { + EchoOperationActionOutcomeV1::Committed(_) => WalTickDecision::Applied, + EchoOperationActionOutcomeV1::Obstructed(_) => WalTickDecision::Obstructed, + EchoOperationActionOutcomeV1::RejectedFootprintConflict { .. } => { + WalTickDecision::RejectedFootprintConflict + } + } +} + fn operation_installation_from_transaction( transaction: &crate::causal_wal::WalRecoveredTransaction, ) -> Result<(InstalledEchoOperationV1, Vec), TrustedRuntimeWalError> { diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 72a34517..52a2c0dd 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -14,8 +14,8 @@ use echo_edict_canonical::{decode_canonical_cbor_v1, encode_canonical_cbor_v1, C use warp_core::causal_wal::{ AffectedFrontier, AffectedFrontierKind, FilesystemWalFaultPlan, FilesystemWalFaultTarget, Lsn, PayloadCodecId, PayloadSchemaId, WalAppendAuthority, WalBuildError, WalDurabilityMode, - WalRecordKind, WalSegmentId, WalTransactionBuilder, WalTransactionId, WalTransactionKind, - WalValidationError, WriterEpochId, + WalRecordKind, WalSegmentId, WalTickDecision, WalTransactionBuilder, WalTransactionId, + WalTransactionKind, WalValidationError, WriterEpochId, }; use warp_core::{ make_head_id, make_node_id, make_type_id, make_warp_id, AtomPayload, AttachmentValue, @@ -2841,6 +2841,15 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { missing_installation.validate_echo_operation_action_outcomes_for_test(), Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) )); + let mut contradictory_decision = adversarial.clone(); + contradictory_decision.replace_echo_operation_action_decision_for_test( + first_submission_id, + WalTickDecision::Obstructed, + ); + assert!(matches!( + contradictory_decision.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); let mut forged_composition = adversarial.clone(); let Some((_, _, EchoOperationActionOutcomeV1::Committed(receipt))) = forged_composition .echo_operation_action_outcomes From 57ac642d605ee560b3eff67752a9e7a94abf8fa0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:34:43 -0700 Subject: [PATCH 16/64] Fix: attribute executable Action outcomes by position --- crates/warp-core/src/coordinator.rs | 70 ++++++++++-------- .../executable_operation_pipeline_tests.rs | 74 +++++++++++++++++++ 2 files changed, 115 insertions(+), 29 deletions(-) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 2488d3f2..f713a959 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -1747,6 +1747,40 @@ impl WorldlineRuntime { .iter() .enumerate() .filter(|(_idx, entry)| entry.scope.local_id == ingress_node); + let positional_candidate = || { + let current_basis = receipt_correlation_current_basis(correlation); + let ticketed_ingress_ids = self + .receipt_correlations_by_current_basis + .get(¤t_basis)?; + let mut same_tick = ticketed_ingress_ids + .iter() + .filter_map(|ticketed_ingress_id| { + self.receipt_correlations_by_ticketed_ingress + .get(ticketed_ingress_id) + }) + .collect::>(); + same_tick.sort_by_key(|candidate| candidate.ingress_id); + if same_tick.len() != receipt.entries().len() + || same_tick + .windows(2) + .any(|pair| pair[0].ingress_id >= pair[1].ingress_id) + { + return None; + } + let index = same_tick.iter().position(|candidate| { + candidate.ticketed_ingress_id == correlation.ticketed_ingress_id + })?; + receipt.entries().get(index).map(|entry| (index, entry)) + }; + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let is_echo_operation_action = self + .witnessed_submission_envelopes + .get(&correlation.submission_id) + .is_some_and(|envelope| { + crate::echo_operation::echo_operation_action_invocation_bytes_v1(envelope).is_some() + }); + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + let is_echo_operation_action = false; let candidate = if let Some(provider) = correlation .contract .as_ref() @@ -1762,43 +1796,21 @@ impl WorldlineRuntime { return no_match(); } candidate + } else if is_echo_operation_action { + let Some(candidate) = positional_candidate() else { + return no_match(); + }; + candidate } else if let Some(candidate) = candidates.next() { if candidates.next().is_some() { return no_match(); } candidate } else { - let current_basis = receipt_correlation_current_basis(correlation); - let Some(ticketed_ingress_ids) = self - .receipt_correlations_by_current_basis - .get(¤t_basis) - else { + let Some(candidate) = positional_candidate() else { return no_match(); }; - let mut same_tick = ticketed_ingress_ids - .iter() - .filter_map(|ticketed_ingress_id| { - self.receipt_correlations_by_ticketed_ingress - .get(ticketed_ingress_id) - }) - .collect::>(); - same_tick.sort_by_key(|candidate| candidate.ingress_id); - if same_tick.len() != receipt.entries().len() - || same_tick - .windows(2) - .any(|pair| pair[0].ingress_id >= pair[1].ingress_id) - { - return no_match(); - } - let Some(index) = same_tick.iter().position(|candidate| { - candidate.ticketed_ingress_id == correlation.ticketed_ingress_id - }) else { - return no_match(); - }; - let Some(entry) = receipt.entries().get(index) else { - return no_match(); - }; - (index, entry) + candidate }; let (idx, entry) = candidate; let Ok(receipt_entry_index) = u32::try_from(idx) else { diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 52a2c0dd..e1185dac 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3237,6 +3237,80 @@ fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { assert_eq!(recovered_node, node); } +#[test] +fn action_outcome_attribution_ignores_application_controlled_scope_collisions() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the scope-collision fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let valid_invocation = + action_invocation(&host, &installed, head_key, node, b"before", b"after"); + let valid_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + valid_invocation, + ) + .expect("the valid invocation becomes an Action"); + let colliding_scope = NodeKey { + warp_id: node.warp_id, + local_id: warp_core::NodeId(valid_envelope.ingress_id()), + }; + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis()) + .expect("Echo resolves the current exact parent basis"); + let obstructed_invocation = + EchoOperationInvocationV1::anchored_node_attachment_compare_and_set( + installed.package_id(), + installed.operation_coordinate(), + evaluation_basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + colliding_scope, + warp_core::echo_operation_atom_value_digest_v1( + make_type_id("operation-fixture-atom"), + b"before", + ), + b"must-not-appear".to_vec(), + ) + .to_canonical_bytes() + .expect("the scope-collision invocation is canonical"); + let obstructed_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + obstructed_invocation, + ) + .expect("the scope-collision invocation becomes an Action"); + + let valid_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(valid_envelope) + .expect("the valid Action is durable") + .submission_id; + let obstructed_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(obstructed_envelope) + .expect("the obstructed Action is durable") + .submission_id; + + let steps = host + .tick_once() + .expect("scope values cannot change positional Action attribution"); + assert_eq!(steps.len(), 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&obstructed_submission_id) + else { + panic!("the colliding scope still receives its own typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::NodeMissing + ); +} + #[test] fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { let wal_dir = TempWalDir::new(); From b7503a34f26de9cf37fed1a84ddf04b04f12ff2f Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:36:59 -0700 Subject: [PATCH 17/64] Fix: key executable Action outcomes by submission --- crates/warp-core/src/coordinator.rs | 19 ++- crates/warp-core/src/echo_operation.rs | 11 +- crates/warp-core/src/trusted_runtime_host.rs | 30 ++--- .../executable_operation_pipeline_tests.rs | 117 ++++++++++++++++++ 4 files changed, 153 insertions(+), 24 deletions(-) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index f713a959..e1e990fb 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -1402,6 +1402,17 @@ impl WorldlineRuntime { .filter_map(|submission_id| self.witnessed_submissions.get(submission_id)) } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn witnessed_submission_id_for_target( + &self, + head_key: WriterHeadKey, + ingress_id: Hash, + ) -> Option { + self.submission_by_target + .get(&(head_key, ingress_id)) + .copied() + } + /// Returns the number of witnessed submissions without a receipt decision. #[must_use] pub fn pending_witnessed_submission_count(&self) -> usize { @@ -4254,7 +4265,12 @@ impl SchedulerCoordinator { let mut candidates = Vec::with_capacity(admitted.len()); for envelope in &admitted { let ingress_id = envelope.ingress_id(); - let admitted_action = admitted_actions.get(&ingress_id).ok_or( + let submission_id = runtime + .witnessed_submission_id_for_target(*key, ingress_id) + .ok_or(RuntimeError::EchoOperationActionAdmissionMissing( + ingress_id, + ))?; + let admitted_action = admitted_actions.get(&submission_id).ok_or( RuntimeError::EchoOperationActionAdmissionMissing(ingress_id), )?; let current_basis = resolve_echo_operation_evaluation_basis_v1( @@ -4278,6 +4294,7 @@ impl SchedulerCoordinator { evaluation_authority, ); candidates.push(SchedulerEchoOperationCandidateV1 { + submission_id, ingress_id, scope: admitted_action.scope(), rule_id: admitted_action.installed_operation_id().as_hash(), diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 800d0173..d1b89978 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -4150,6 +4150,7 @@ pub(crate) struct EchoOperationCommitMaterialV1 { } pub(crate) struct SchedulerEchoOperationCandidateV1 { + pub submission_id: Hash, pub ingress_id: Hash, pub scope: NodeKey, pub rule_id: Hash, @@ -4235,7 +4236,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( }); blocked_by.push(Vec::new()); decisions.push(( - candidate.ingress_id, + candidate.submission_id, SchedulerEchoOperationDecisionV1::Obstructed(obstruction), )); } @@ -4264,7 +4265,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( out_slots.extend_from_slice(prepared.patch().out_slots()); ops.extend_from_slice(prepared.patch().ops()); decisions.push(( - candidate.ingress_id, + candidate.submission_id, SchedulerEchoOperationDecisionV1::Applied(prepared), )); } else { @@ -4278,7 +4279,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( }); blocked_by.push(blockers.clone()); decisions.push(( - candidate.ingress_id, + candidate.submission_id, SchedulerEchoOperationDecisionV1::RejectedFootprintConflict { installed_operation_id: prepared.installed_operation_id(), invocation_id: prepared.invocation_id(), @@ -4343,7 +4344,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( let outcomes = decisions .into_iter() - .map(|(ingress_id, decision)| { + .map(|(submission_id, decision)| { let outcome = match decision { SchedulerEchoOperationDecisionV1::Applied(prepared) => { let mut receipt = build_receipt( @@ -4379,7 +4380,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( blocked_by, }, }; - (ingress_id, outcome) + (submission_id, outcome) }) .collect(); diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 72b0ea14..2139c6d7 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -1663,7 +1663,7 @@ impl TrustedRuntimeHost { )?; self.echo_operation_action_admission_obstructions .remove(&submission.submission_id); - admitted_actions.insert(submission.ingress_id, admitted); + admitted_actions.insert(submission.submission_id, admitted); } Ok(admitted_actions) } @@ -1704,10 +1704,6 @@ impl TrustedRuntimeHost { .filter(|correlation| !existing_correlations.contains(&correlation.ticketed_ingress_id)) .cloned() .collect::>(); - let action_submission_by_ingress = new_correlations - .iter() - .map(|correlation| (correlation.ingress_id, correlation.submission_id)) - .collect::>(); let mut tick_wal_records = Vec::new(); if self.runtime_wal.is_some() { for correlation in new_correlations { @@ -1737,7 +1733,7 @@ impl TrustedRuntimeHost { tick_wal_records.push((correlation, decision, state_delta, state_delta_digest)); } } - let action_outcomes_by_ingress = + let action_outcomes_by_submission = action_outcomes.iter().cloned().collect::>(); let mut tick_wal_groups = BTreeMap::new(); for (correlation, decision, state_delta, state_delta_digest) in tick_wal_records { @@ -1771,7 +1767,7 @@ impl TrustedRuntimeHost { continue; }; let group_has_action_outcomes = group.iter().any(|(correlation, _, _, _)| { - action_outcomes_by_ingress.contains_key(&correlation.ingress_id) + action_outcomes_by_submission.contains_key(&correlation.submission_id) }); let result = if group.len() == 1 && !group_has_action_outcomes { runtime_wal.record_tick_receipt( @@ -1787,7 +1783,7 @@ impl TrustedRuntimeHost { .collect::>(); runtime_wal.record_tick_receipt_batch( &correlations, - &action_outcomes_by_ingress, + &action_outcomes_by_submission, state_delta, *state_delta_digest, ) @@ -1806,11 +1802,9 @@ impl TrustedRuntimeHost { } } } - for (ingress_id, outcome) in action_outcomes { - if let Some(submission_id) = action_submission_by_ingress.get(&ingress_id) { - self.echo_operation_action_outcomes - .insert(*submission_id, outcome); - } + for (submission_id, outcome) in action_outcomes { + self.echo_operation_action_outcomes + .insert(submission_id, outcome); } Ok(records) } @@ -2579,7 +2573,7 @@ impl TrustedRuntimeWal { fn record_tick_receipt_batch( &mut self, correlations: &[(ReceiptCorrelationRecord, WalTickDecision)], - action_outcomes_by_ingress: &BTreeMap, + action_outcomes_by_submission: &BTreeMap, state_delta: &WalRuntimeStateDeltaRecord, state_delta_digest: Hash, ) -> Result { @@ -2605,9 +2599,9 @@ impl TrustedRuntimeWal { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } let first_has_action_outcome = - action_outcomes_by_ingress.contains_key(&first_correlation.ingress_id); + action_outcomes_by_submission.contains_key(&first_correlation.submission_id); if correlations.iter().any(|(correlation, _)| { - action_outcomes_by_ingress.contains_key(&correlation.ingress_id) + action_outcomes_by_submission.contains_key(&correlation.submission_id) != first_has_action_outcome }) { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); @@ -2626,8 +2620,8 @@ impl TrustedRuntimeWal { }; next_receipt_frontier = receipt_frontier_digest(next_receipt_frontier, receipt, &wal_correlation); - let action_outcome = action_outcomes_by_ingress - .get(&correlation.ingress_id) + let action_outcome = action_outcomes_by_submission + .get(&correlation.submission_id) .map(|outcome| { if wal_tick_decision_for_action_outcome(outcome) != *decision { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index e1185dac..aa0a5ae3 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -159,6 +159,71 @@ fn fixture_host() -> (TrustedRuntimeHost, WriterHeadKey, NodeKey) { (host, head_key, node) } +fn fixture_host_with_two_worldlines() -> (TrustedRuntimeHost, WriterHeadKey, WriterHeadKey, NodeKey) +{ + let warp_id = make_warp_id("operation-fixture"); + let node_id = make_node_id("operation-fixture-root"); + let node = NodeKey { + warp_id, + local_id: node_id, + }; + let mut store = GraphStore::new(warp_id); + store.insert_node( + node_id, + NodeRecord { + ty: make_type_id("operation-fixture-node"), + }, + ); + store.set_node_attachment( + node_id, + Some(AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))), + ); + let state = WorldlineState::from_root_store(store, node_id) + .expect("the fixture state has one lawful root"); + let first_head = WriterHeadKey { + worldline_id: WorldlineId::from_bytes(digest("operation-fixture-worldline-first")), + head_id: make_head_id("operation-fixture-writer"), + }; + let second_head = WriterHeadKey { + worldline_id: WorldlineId::from_bytes(digest("operation-fixture-worldline-second")), + head_id: make_head_id("operation-fixture-writer"), + }; + let mut runtime = WorldlineRuntime::new(); + for head in [first_head, second_head] { + runtime + .register_worldline(head.worldline_id, state.clone()) + .expect("the fixture worldline registers"); + runtime + .register_writer_head(WriterHead::with_routing( + head, + PlaybackMode::Play, + InboxPolicy::AcceptAll, + None, + true, + )) + .expect("the fixture writer registers"); + } + + let mut engine_store = GraphStore::default(); + let engine_root = make_node_id("root"); + engine_store.insert_node( + engine_root, + NodeRecord { + ty: make_type_id("world"), + }, + ); + let engine = EngineBuilder::new(engine_store, engine_root) + .scheduler(SchedulerKind::Radix) + .workers(1) + .build(); + let host = TrustedRuntimeHost::new(runtime, engine) + .expect("the trusted Echo runtime host initializes"); + (host, first_head, second_head, node) +} + /// Like [`fixture_host`], but the warp-scoped store also contains a second, /// bare node -- present, but with no alpha attachment set -- at /// `second_node_local_id`, typed `second_node_type`. Used to exercise @@ -3311,6 +3376,58 @@ fn action_outcome_attribution_ignores_application_controlled_scope_collisions() ); } +#[test] +fn identical_action_payloads_on_distinct_heads_keep_distinct_outcomes() { + let (mut host, first_head, second_head, node) = fixture_host_with_two_worldlines(); + host.enable_in_memory_runtime_wal() + .expect("the cross-head fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let invocation = action_invocation(&host, &installed, first_head, node, b"before", b"after"); + let first_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: first_head }, + invocation.clone(), + ) + .expect("the first target accepts the canonical Action payload"); + let second_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: second_head }, + invocation, + ) + .expect("the second target accepts the same canonical Action payload"); + assert_eq!(first_envelope.ingress_id(), second_envelope.ingress_id()); + + let first_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(first_envelope) + .expect("the first target submission is durable") + .submission_id; + let second_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(second_envelope) + .expect("the second target submission is durable") + .submission_id; + assert_ne!(first_submission_id, second_submission_id); + + let steps = host + .tick_once() + .expect("cross-head duplicate payloads retain distinct Action identities"); + assert_eq!(steps.len(), 2); + assert!(matches!( + host.echo_operation_action_outcome_v1(&first_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&second_submission_id) + else { + panic!("the mismatched target retains its own typed basis obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); +} + #[test] fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { let wal_dir = TempWalDir::new(); From 37fe6181342e529dd5aa2b1d46577c4a475e24e4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:38:56 -0700 Subject: [PATCH 18/64] Fix: key recovered Action Ticks by causal coordinate --- crates/warp-core/src/trusted_runtime_host.rs | 34 +++++++++++--- .../executable_operation_pipeline_tests.rs | 46 +++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 2139c6d7..d008af0c 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -4111,10 +4111,25 @@ fn validate_recovered_echo_operation_action_outcomes( .collect::>(); let provenance = provenance_entries .iter() - .map(|entry| (entry.expected.commit_hash, entry)) + .filter_map(|entry| { + Some(( + ( + entry.head_key?, + entry.worldline_tick.checked_add(1)?, + entry.commit_global_tick, + entry.expected.commit_hash, + ), + entry, + )) + }) .collect::>(); - let mut outcomes_by_commit = BTreeMap::< - Hash, + let mut outcomes_by_tick = BTreeMap::< + ( + crate::WriterHeadKey, + crate::WorldlineTick, + crate::GlobalTick, + Hash, + ), Vec<( Hash, &EchoOperationActionOutcomeV1, @@ -4184,14 +4199,19 @@ fn validate_recovered_echo_operation_action_outcomes( } } } - outcomes_by_commit - .entry(correlation.commit_hash) + outcomes_by_tick + .entry(( + correlation.head_key, + correlation.worldline_tick_after, + correlation.commit_global_tick, + correlation.commit_hash, + )) .or_default() .push((*ingress_id, outcome, correlation, invocation)); } - for (commit_hash, mut group) in outcomes_by_commit { + for (tick_coordinate, mut group) in outcomes_by_tick { let entry = provenance - .get(&commit_hash) + .get(&tick_coordinate) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; let tick_receipt = entry .tick_receipt diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index aa0a5ae3..36c46e14 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3428,6 +3428,52 @@ fn identical_action_payloads_on_distinct_heads_keep_distinct_outcomes() { ); } +#[test] +fn recovery_separates_identical_commits_on_distinct_worldlines() { + let (mut host, first_head, second_head, node) = fixture_host_with_two_worldlines(); + host.enable_in_memory_runtime_wal() + .expect("the multi-worldline recovery fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let mut submission_ids = Vec::new(); + for head_key in [first_head, second_head] { + let invocation = action_invocation(&host, &installed, head_key, node, b"before", b"after"); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the exact-head invocation becomes an Action"); + submission_ids.push( + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the exact-head Action is durable") + .submission_id, + ); + } + + let steps = host + .tick_once() + .expect("both exact-head Actions commit in one scheduler pass"); + assert_eq!(steps.len(), 2); + assert_eq!(steps[0].commit_hash, steps[1].commit_hash); + + let recovery = host + .runtime_wal() + .expect("the scheduler WAL remains enabled") + .recover_read_only() + .expect("equal commit hashes on distinct worldlines recover independently"); + for submission_id in submission_ids { + assert!(recovery + .echo_operation_action_outcomes + .iter() + .any(|(recovered, _, outcome)| { + *recovered == submission_id + && matches!(outcome, EchoOperationActionOutcomeV1::Committed(_)) + })); + } +} + #[test] fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { let wal_dir = TempWalDir::new(); From 1f7c5c1487c7210e91db3e47fb9b730831c7fdd7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:40:48 -0700 Subject: [PATCH 19/64] Fix: bind recovered Action admission tickets --- crates/warp-core/src/trusted_runtime_host.rs | 50 +++++++++++++++++-- .../executable_operation_pipeline_tests.rs | 10 ++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index d008af0c..26b1d3d5 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -4105,6 +4105,11 @@ fn validate_recovered_echo_operation_action_outcomes( .iter() .map(|record| (record.submission.submission_id, &record.envelope)) .collect::>(); + let submissions = witnessed_submissions + .records() + .iter() + .map(|record| (record.submission.submission_id, &record.submission)) + .collect::>(); let correlations = receipt_correlations .iter() .map(|correlation| (correlation.submission_id, correlation)) @@ -4144,6 +4149,9 @@ fn validate_recovered_echo_operation_action_outcomes( let envelope = envelopes .get(submission_id) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let submission = submissions + .get(submission_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; let correlation = correlations .get(submission_id) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; @@ -4158,6 +4166,28 @@ fn validate_recovered_echo_operation_action_outcomes( if envelope.ingress_id() != *ingress_id { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } + let invocation_admission_id = match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + Some(receipt.invocation_admission_id()) + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + Some(obstruction.invocation_admission_id()) + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict { .. } => None, + }; + if correlation.ticket_digest != correlation.causal_receipt_ref.ticket_digest + || invocation_admission_id.is_some_and(|admission_id| { + correlation.ticket_digest + != echo_operation_action_admission_digest_from_parts( + submission, + invocation.package_id, + installed.installed_operation_id(), + admission_id, + ) + }) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } match outcome { EchoOperationActionOutcomeV1::Committed(receipt) => { if validate_receipt_installation_v1(receipt, installed).is_err() @@ -5086,6 +5116,20 @@ fn provider_contract_host_admission_digest( fn echo_operation_action_admission_digest( submission: &IntentSubmissionRecord, admitted: &AdmittedEchoOperationInvocationV1, +) -> Hash { + echo_operation_action_admission_digest_from_parts( + submission, + admitted.package_id(), + admitted.installed_operation_id(), + admitted.admission_id(), + ) +} + +fn echo_operation_action_admission_digest_from_parts( + submission: &IntentSubmissionRecord, + package_id: crate::EchoOperationPackageIdV1, + installed_operation_id: crate::InstalledEchoOperationIdV1, + invocation_admission_id: crate::EchoOperationInvocationAdmissionIdV1, ) -> Hash { let mut hasher = blake3::Hasher::new(); hasher.update(ECHO_OPERATION_ACTION_ADMISSION_DIGEST_DOMAIN); @@ -5093,9 +5137,9 @@ fn echo_operation_action_admission_digest( hasher.update(&submission.ingress_id); hasher.update(submission.head_key.worldline_id.as_bytes()); hasher.update(submission.head_key.head_id.as_bytes()); - hasher.update(&admitted.package_id().as_hash()); - hasher.update(&admitted.installed_operation_id().as_hash()); - hasher.update(&admitted.admission_id().as_hash()); + hasher.update(&package_id.as_hash()); + hasher.update(&installed_operation_id.as_hash()); + hasher.update(&invocation_admission_id.as_hash()); hasher.finalize().into() } diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 36c46e14..0bdf86c1 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2915,6 +2915,16 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { contradictory_decision.validate_echo_operation_action_outcomes_for_test(), Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) )); + let mut forged_admission_ticket = adversarial.clone(); + let forged_ticket = digest("forged-Action-admission-ticket"); + forged_admission_ticket.receipt_correlations[0].ticket_digest = forged_ticket; + forged_admission_ticket.receipt_correlations[0] + .causal_receipt_ref + .ticket_digest = forged_ticket; + assert!(matches!( + forged_admission_ticket.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); let mut forged_composition = adversarial.clone(); let Some((_, _, EchoOperationActionOutcomeV1::Committed(receipt))) = forged_composition .echo_operation_action_outcomes From 29547d4964930907af2b3eb11fb3b315ee87c2d8 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:42:16 -0700 Subject: [PATCH 20/64] Fix: enforce Action installation order during recovery --- crates/warp-core/src/trusted_runtime_host.rs | 37 +++++++++++++++++++ .../executable_operation_pipeline_tests.rs | 7 ++++ 2 files changed, 44 insertions(+) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 26b1d3d5..9a1f6701 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -453,6 +453,8 @@ pub struct TrustedRuntimeWalRecovery { /// scheduler-owned Tick records, keyed by witnessed submission. pub echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, echo_operation_action_decisions: BTreeMap, + echo_operation_action_installations_before_tick: + BTreeMap>, causal_history_frontiers: Vec, } @@ -470,6 +472,7 @@ impl TrustedRuntimeWalRecovery { &self.installed_echo_operations, &self.echo_operation_action_outcomes, &self.echo_operation_action_decisions, + &self.echo_operation_action_installations_before_tick, ) } @@ -484,6 +487,16 @@ impl TrustedRuntimeWalRecovery { .insert(submission_id, decision); } + /// Removes the installation-order witness for one Action in adversarial tests. + #[cfg(any(test, feature = "host_test"))] + pub fn clear_echo_operation_action_installations_before_tick_for_test( + &mut self, + submission_id: Hash, + ) { + self.echo_operation_action_installations_before_tick + .remove(&submission_id); + } + /// Recomputes the certificate's canonical index root from recovered evidence. /// /// # Errors @@ -2215,6 +2228,8 @@ impl TrustedRuntimeWal { let receipt_correlations = runtime_state.receipt_correlations; let echo_operation_action_outcomes = runtime_state.echo_operation_action_outcomes; let echo_operation_action_decisions = runtime_state.echo_operation_action_decisions; + let echo_operation_action_installations_before_tick = + runtime_state.echo_operation_action_installations_before_tick; let missing_runtime_state_deltas = runtime_state.missing_runtime_state_deltas; let installed_echo_operations = operation_material.installations; let echo_operation_receipts = operation_material.receipts; @@ -2226,6 +2241,7 @@ impl TrustedRuntimeWal { &installed_echo_operations, &echo_operation_action_outcomes, &echo_operation_action_decisions, + &echo_operation_action_installations_before_tick, )?; let certificate = runtime_wal_recovery_certificate( &report, @@ -2256,6 +2272,7 @@ impl TrustedRuntimeWal { echo_operation_receipts, echo_operation_action_outcomes, echo_operation_action_decisions, + echo_operation_action_installations_before_tick, causal_history_frontiers, }) } @@ -3902,6 +3919,8 @@ struct RecoveredRuntimeStateMaterial { receipt_correlations: Vec, echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, echo_operation_action_decisions: BTreeMap, + echo_operation_action_installations_before_tick: + BTreeMap>, missing_runtime_state_deltas: Vec, } @@ -3912,9 +3931,18 @@ fn recover_runtime_state_delta_material( let mut correlations_by_submission = BTreeMap::new(); let mut action_outcomes_by_submission = BTreeMap::new(); let mut action_decisions_by_submission = BTreeMap::new(); + let mut action_installations_before_tick = BTreeMap::new(); + let mut installed_packages = BTreeSet::new(); let mut submission_by_ticket = BTreeMap::new(); let mut missing = Vec::new(); for transaction in &report.transactions { + if transaction.commit.transaction_kind + == WalTransactionKind::ExecutableOperationInstallation + { + let (installed, _) = operation_installation_from_transaction(transaction)?; + installed_packages.insert(installed.package_id()); + continue; + } if transaction.commit.transaction_kind == WalTransactionKind::ExecutableOperationTick { let (_, state_delta, _) = operation_tick_records_from_transaction(transaction)?; let entry = state_delta.provenance_entry().clone(); @@ -4021,6 +4049,7 @@ fn recover_runtime_state_delta_material( return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } action_outcomes_by_submission.insert(submission_id, (ingress_id, outcome)); + action_installations_before_tick.insert(submission_id, installed_packages.clone()); } } let coordinate = (entry.worldline_id, entry.worldline_tick); @@ -4063,6 +4092,7 @@ fn recover_runtime_state_delta_material( receipt_correlations: correlations, echo_operation_action_outcomes, echo_operation_action_decisions: action_decisions_by_submission, + echo_operation_action_installations_before_tick: action_installations_before_tick, missing_runtime_state_deltas: missing, }) } @@ -4099,6 +4129,7 @@ fn validate_recovered_echo_operation_action_outcomes( installed_echo_operations: &[InstalledEchoOperationV1], outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], decisions: &BTreeMap, + installations_before_tick: &BTreeMap>, ) -> Result<(), TrustedRuntimeWalError> { let envelopes = witnessed_submissions .records() @@ -4163,6 +4194,12 @@ fn validate_recovered_echo_operation_action_outcomes( .iter() .find(|installed| installed.package_id() == invocation.package_id) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + if installations_before_tick + .get(submission_id) + .is_none_or(|packages| !packages.contains(&invocation.package_id)) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } if envelope.ingress_id() != *ingress_id { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 0bdf86c1..8f82f160 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2925,6 +2925,13 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { forged_admission_ticket.validate_echo_operation_action_outcomes_for_test(), Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) )); + let mut retroactive_installation = adversarial.clone(); + retroactive_installation + .clear_echo_operation_action_installations_before_tick_for_test(first_submission_id); + assert!(matches!( + retroactive_installation.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); let mut forged_composition = adversarial.clone(); let Some((_, _, EchoOperationActionOutcomeV1::Committed(receipt))) = forged_composition .echo_operation_action_outcomes From 48c99b1d08716c0c8fab9836da1031af250688e4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:43:25 -0700 Subject: [PATCH 21/64] Fix: require durable Action acceptance before admission --- crates/warp-core/src/trusted_runtime_host.rs | 7 +++ .../executable_operation_pipeline_tests.rs | 49 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 9a1f6701..52aae63a 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -1640,6 +1640,13 @@ impl TrustedRuntimeHost { }; let mut admitted_actions = BTreeMap::new(); for (submission, envelope, invocation_bytes) in pending { + if let Some(runtime_wal) = self.runtime_wal.as_ref() { + let durably_accepted = runtime_wal + .has_submission_acceptance(submission.submission_id, submission.ingress_id)?; + if !durably_accepted { + continue; + } + } if self .echo_operation_action_admission_obstructions .contains_key(&submission.submission_id) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 8f82f160..a7b29928 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3100,6 +3100,55 @@ fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { ); } +#[test] +fn wal_enabled_host_does_not_auto_admit_non_durable_action_submission() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the non-durable submission fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"must-not-run", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes an Action"); + let submission_id = host + .app() + .submit_intent(envelope) + .expect("ordinary intake records process-local witnessed history") + .submission_id; + + let steps = host + .tick_once() + .expect("non-durable intake cannot enter the scheduler"); + assert!(steps.is_empty()); + assert!(host + .echo_operation_action_outcome_v1(&submission_id) + .is_none()); + assert!(matches!( + host.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + let recovery = host + .runtime_wal() + .expect("the runtime WAL remains enabled") + .recover_read_only() + .expect("the WAL contains no dangling scheduler consequence"); + assert!(recovery.witnessed_submissions.is_empty()); + assert!(recovery.echo_operation_action_outcomes.is_empty()); +} + #[test] fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { let wal_dir = TempWalDir::new(); From 862721a919f4f5d84d9b3d3e2d3f3f75c38df65a Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 08:46:50 -0700 Subject: [PATCH 22/64] Fix: corroborate recovered Action evaluation bases --- crates/warp-core/src/echo_operation.rs | 16 +++ crates/warp-core/src/trusted_runtime_host.rs | 120 +++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index d1b89978..be90ba30 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -2540,6 +2540,22 @@ fn current_application_basis( } } +pub(crate) fn action_application_basis_matches_state_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + state: &WorldlineState, +) -> Result { + let invocation = EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) + .map_err(|error| { + invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::MalformedInvocation, + error.to_string(), + ) + })?; + Ok(current_application_basis(installed, &invocation, state)? + == invocation.evaluation_basis.application_basis) +} + pub(crate) fn decode_invocation_route_v1( canonical_invocation_bytes: &[u8], ) -> Result< diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 52aae63a..eba4aaee 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -42,6 +42,7 @@ use crate::{ }, contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ + action_application_basis_matches_state_v1, action_batch_composition_digest_from_receipts_v1, admit_action_invocation_v1, admit_invocation_v1, admit_package_v1, commit_prepared_to_state, decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, @@ -3804,9 +3805,116 @@ fn validate_recovered_echo_operation_parent_states( }); } } + + let submissions = recovery + .witnessed_submissions + .records() + .iter() + .map(|record| (record.submission.submission_id, record)) + .collect::>(); + let correlations = recovery + .receipt_correlations + .iter() + .map(|correlation| (correlation.submission_id, correlation)) + .collect::>(); + for (submission_id, _, outcome) in &recovery.echo_operation_action_outcomes { + let submission = submissions.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no retained submission", + }, + )?; + let correlation = correlations.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no retained receipt correlation", + }, + )?; + let invocation_bytes = echo_operation_action_invocation_bytes_v1(&submission.envelope) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no canonical invocation", + })?; + let invocation = inspect_action_invocation_v1(invocation_bytes).map_err(|_| { + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome invocation cannot be inspected", + } + })?; + let installed = installations.get(&invocation.package_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no recovered installation", + }, + )?; + let basis = invocation.evaluation_basis; + let worldline_id = basis.writer_head().worldline_id; + let frontier = runtime.worldlines().get(&worldline_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis names an unavailable recovery worldline", + }, + )?; + let basis_state = recovered_provenance + .replay_worldline_state_at(worldline_id, frontier.state(), basis.worldline_tick()) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis state cannot be reconstructed", + })?; + if basis_state.state_root() != basis.state_root() + || !action_application_basis_matches_state_v1(installed, invocation_bytes, &basis_state) + .unwrap_or(false) + || !evaluation_basis_matches_recovered_coordinate(basis, &entries) + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis disagrees with reconstructed causal state", + }); + } + let tick_before = correlation + .worldline_tick_after + .as_u64() + .checked_sub(1) + .map(crate::WorldlineTick::from_raw) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has an impossible Tick coordinate", + })?; + let basis_is_current = + basis.writer_head() == correlation.head_key && basis.worldline_tick() == tick_before; + let basis_changed = matches!( + outcome, + EchoOperationActionOutcomeV1::Obstructed(obstruction) + if obstruction.kind() == crate::EchoOperationObstructionKindV1::BasisChanged + ); + if basis_changed == basis_is_current { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis posture disagrees with the scheduler Tick parent", + }); + } + } Ok(()) } +fn evaluation_basis_matches_recovered_coordinate( + basis: EchoOperationEvaluationBasisV1, + entries: &BTreeMap<(crate::WorldlineId, crate::WorldlineTick), &ProvenanceEntry>, +) -> bool { + if basis.worldline_tick() == crate::WorldlineTick::ZERO { + return basis.commit_global_tick().is_none() + && basis.commit_id() + == crate::echo_operation::genesis_commit_id( + basis.writer_head(), + basis.state_root(), + ); + } + let Some(parent_tick) = basis.worldline_tick().as_u64().checked_sub(1) else { + return false; + }; + entries + .get(&( + basis.writer_head().worldline_id, + crate::WorldlineTick::from_raw(parent_tick), + )) + .is_some_and(|parent| { + parent.head_key == Some(basis.writer_head()) + && parent.expected.state_root == basis.state_root() + && parent.expected.commit_hash == basis.commit_id() + && basis.commit_global_tick() == Some(parent.commit_global_tick) + }) +} + fn validate_operation_receipt_state_delta( receipt: &EchoOperationReceiptV1, state_delta: &WalRuntimeStateDeltaRecord, @@ -5830,16 +5938,28 @@ mod tests { let provenance = BTreeMap::from([(parent_coordinate, &parent)]); validate_operation_receipt_parent_material(basis, &child, &provenance) .expect("the exact retained parent corroborates every causal basis field"); + assert!(evaluation_basis_matches_recovered_coordinate( + basis, + &provenance + )); let mut wrong_root = parent.clone(); wrong_root.expected.state_root = [39; 32]; let provenance = BTreeMap::from([(parent_coordinate, &wrong_root)]); assert!(validate_operation_receipt_parent_material(basis, &child, &provenance).is_err()); + assert!(!evaluation_basis_matches_recovered_coordinate( + basis, + &provenance + )); let mut wrong_global_tick = parent.clone(); wrong_global_tick.commit_global_tick = GlobalTick::from_raw(9); let provenance = BTreeMap::from([(parent_coordinate, &wrong_global_tick)]); assert!(validate_operation_receipt_parent_material(basis, &child, &provenance).is_err()); + assert!(!evaluation_basis_matches_recovered_coordinate( + basis, + &provenance + )); } fn test_correlation(receipt_digest: Hash) -> ReceiptCorrelationRecord { From 72eafc8a7f972ab88b2dc922a45178143f5787ea Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:06:34 -0700 Subject: [PATCH 23/64] Fix: reconstruct Action footprint conflicts during recovery --- crates/warp-core/src/echo_operation.rs | 388 +++++++++++++++--- crates/warp-core/src/lib.rs | 23 +- crates/warp-core/src/trusted_runtime_host.rs | 255 ++++++++++-- .../executable_operation_pipeline_tests.rs | 82 ++++ 4 files changed, 657 insertions(+), 91 deletions(-) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index be90ba30..b552f875 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -84,7 +84,7 @@ const ACTION_BATCH_RULE_PACK_DOMAIN: &[u8] = b"echo:operation-action-batch-rule- const ACTION_BATCH_PLAN_DOMAIN: &[u8] = b"echo:operation-action-batch-plan:v1\0"; const ACTION_BATCH_REWRITES_DOMAIN: &[u8] = b"echo:operation-action-batch-rewrites:v1\0"; const ACTION_BATCH_COMPOSITION_DOMAIN: &[u8] = b"echo:operation-action-batch-composition:v1\0"; -const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT001"; +const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT002"; const INVOCATION_BYTES_DIGEST_DOMAIN: &[u8] = b"echo:operation-invocation-bytes:v1\0"; const BASIS_ID_DOMAIN: &[u8] = b"echo:operation-evaluation-basis:v1\0"; const PACKAGE_ADMISSION_ID_DOMAIN: &[u8] = b"echo:operation-package-admission:v1\0"; @@ -2556,6 +2556,98 @@ pub(crate) fn action_application_basis_matches_state_v1( == invocation.evaluation_basis.application_basis) } +pub(crate) fn action_admission_evidence_matches_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + maximum_budget: EchoOperationBudgetV1, + expected_policy_id: Hash, + expected_admission_id: EchoOperationInvocationAdmissionIdV1, +) -> bool { + let Ok(invocation) = + EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) + else { + return false; + }; + let policy = EchoOperationInvocationAdmissionPolicyV1::new( + installed.authority_profile_identity, + invocation.authority_grant_identity, + maximum_budget, + ); + let policy_id = policy.identity(); + policy_id == expected_policy_id + && invocation_admission_id( + installed.installed_operation_id, + EchoOperationInvocationIdV1(domain_hash( + INVOCATION_ID_DOMAIN, + canonical_invocation_bytes, + )), + policy_id, + invocation.evaluation_basis.identity(), + ) == expected_admission_id +} + +pub(crate) fn action_preparation_identity_matches_v1( + private_evaluation_id: EchoOperationPrivateEvaluationIdV1, + prepared_patch_digest: Hash, + prepared_result_id: EchoOperationResultIdV1, + expected_preparation_id: PreparedEchoOperationIdV1, +) -> bool { + preparation_id( + private_evaluation_id, + prepared_patch_digest, + prepared_result_id, + ) == expected_preparation_id +} + +pub(crate) fn reconstruct_action_preparation_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + maximum_budget: EchoOperationBudgetV1, + expected_policy_id: Hash, + expected_admission_id: EchoOperationInvocationAdmissionIdV1, + state: &WorldlineState, + policy_id: u32, +) -> Option> { + if !action_admission_evidence_matches_v1( + installed, + canonical_invocation_bytes, + maximum_budget, + expected_policy_id, + expected_admission_id, + ) { + return None; + } + let invocation = + EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes).ok()?; + let policy = EchoOperationInvocationAdmissionPolicyV1::new( + installed.authority_profile_identity, + invocation.authority_grant_identity, + maximum_budget, + ); + let authority = EchoOperationEvaluationAuthorityV1::new(); + let admitted = admit_action_invocation_v1( + Some(installed), + policy, + canonical_invocation_bytes, + authority.clone(), + ) + .ok()?; + if admitted.admission_id() != expected_admission_id { + return None; + } + match prepare_operation_v1( + Some(installed), + admitted, + invocation.evaluation_basis, + state, + policy_id, + &authority, + ) { + EchoOperationPreparationV1::Prepared(prepared) => Some(prepared), + EchoOperationPreparationV1::Obstructed(_) => None, + } +} + pub(crate) fn decode_invocation_route_v1( canonical_invocation_bytes: &[u8], ) -> Result< @@ -2721,6 +2813,119 @@ impl EchoOperationObstructionV1 { } } +/// Complete retained evidence for a privately prepared Action rejected during +/// scheduler composition. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EchoOperationFootprintConflictV1 { + /// Exact installed package which admitted the rejected preparation. + pub(crate) package_id: EchoOperationPackageIdV1, + /// Exact installed operation whose prepared consequence conflicted. + pub(crate) installed_operation_id: InstalledEchoOperationIdV1, + /// Runtime policy identity used to admit the rejected invocation. + pub(crate) invocation_admission_policy_id: Hash, + /// Runtime policy ceiling used to admit the rejected invocation. + pub(crate) invocation_admission_maximum_budget: EchoOperationBudgetV1, + /// Exact invocation admission consumed by private evaluation. + pub(crate) invocation_admission_id: EchoOperationInvocationAdmissionIdV1, + /// Exact canonical invocation whose prepared consequence conflicted. + pub(crate) invocation_id: EchoOperationInvocationIdV1, + /// Exact evaluation basis consumed by private evaluation. + pub(crate) evaluation_basis_id: EchoOperationEvaluationBasisIdV1, + /// Exact private-evaluation identity produced before composition. + pub(crate) private_evaluation_id: EchoOperationPrivateEvaluationIdV1, + /// Exact prepared patch identity produced before composition. + pub(crate) prepared_patch_digest: Hash, + /// Exact prepared result identity produced before composition. + pub(crate) prepared_result_id: EchoOperationResultIdV1, + /// Exact actual footprint observed during private evaluation. + pub(crate) actual_footprint_digest: Hash, + /// Identity of the privately prepared candidate that was not applied. + pub(crate) preparation_id: PreparedEchoOperationIdV1, + /// Canonical indices of earlier applied Tick members that blocked it. + pub(crate) blocked_by: Vec, +} + +impl EchoOperationFootprintConflictV1 { + /// Returns the exact package which admitted the rejected preparation. + #[must_use] + pub const fn package_id(&self) -> EchoOperationPackageIdV1 { + self.package_id + } + + /// Returns the installed operation whose prepared consequence conflicted. + #[must_use] + pub const fn installed_operation_id(&self) -> InstalledEchoOperationIdV1 { + self.installed_operation_id + } + + /// Returns the runtime policy identity used to admit the invocation. + #[must_use] + pub const fn invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission_policy_id + } + + /// Returns the runtime policy ceiling used to admit the invocation. + #[must_use] + pub const fn invocation_admission_maximum_budget(&self) -> EchoOperationBudgetV1 { + self.invocation_admission_maximum_budget + } + + /// Returns the invocation-admission evidence consumed by evaluation. + #[must_use] + pub const fn invocation_admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { + self.invocation_admission_id + } + + /// Returns the exact canonical invocation which was privately evaluated. + #[must_use] + pub const fn invocation_id(&self) -> EchoOperationInvocationIdV1 { + self.invocation_id + } + + /// Returns the exact private-evaluation basis identity. + #[must_use] + pub const fn evaluation_basis_id(&self) -> EchoOperationEvaluationBasisIdV1 { + self.evaluation_basis_id + } + + /// Returns the bounded private-evaluation identity. + #[must_use] + pub const fn private_evaluation_id(&self) -> EchoOperationPrivateEvaluationIdV1 { + self.private_evaluation_id + } + + /// Returns the rejected preparation's patch digest. + #[must_use] + pub const fn prepared_patch_digest(&self) -> Hash { + self.prepared_patch_digest + } + + /// Returns the rejected preparation's typed result identity. + #[must_use] + pub const fn prepared_result_id(&self) -> EchoOperationResultIdV1 { + self.prepared_result_id + } + + /// Returns the evaluator-recorded actual-footprint identity. + #[must_use] + pub const fn actual_footprint_digest(&self) -> Hash { + self.actual_footprint_digest + } + + /// Returns the complete rejected preparation's identity. + #[must_use] + pub const fn preparation_id(&self) -> PreparedEchoOperationIdV1 { + self.preparation_id + } + + /// Returns canonical indices of earlier applied Tick members which blocked + /// this preparation. + #[must_use] + pub fn blocked_by(&self) -> &[u32] { + &self.blocked_by + } +} + /// Typed terminal outcome for one executable-operation Action selected by a /// scheduler-owned Tick. #[derive(Clone, Debug, PartialEq, Eq)] @@ -2731,16 +2936,23 @@ pub enum EchoOperationActionOutcomeV1 { Obstructed(EchoOperationObstructionV1), /// An earlier applied Action in the same Tick reserved an overlapping /// footprint. - RejectedFootprintConflict { - /// Exact installed operation whose prepared consequence conflicted. - installed_operation_id: InstalledEchoOperationIdV1, - /// Exact canonical invocation whose prepared consequence conflicted. - invocation_id: EchoOperationInvocationIdV1, - /// Identity of the privately prepared candidate that was not applied. - preparation_id: PreparedEchoOperationIdV1, - /// Canonical indices of earlier applied Tick members that blocked it. - blocked_by: Vec, - }, + RejectedFootprintConflict(Box), +} + +impl EchoOperationActionOutcomeV1 { + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_conflict_preparation_id_for_test(&mut self, digest: Hash) { + if let Self::RejectedFootprintConflict(conflict) = self { + conflict.preparation_id = PreparedEchoOperationIdV1(digest); + } + } + + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_conflict_blockers_for_test(&mut self, replacement: Vec) { + if let Self::RejectedFootprintConflict(conflict) = self { + conflict.blocked_by = replacement; + } + } } pub(crate) fn retain_action_outcome_v1( @@ -2770,20 +2982,41 @@ pub(crate) fn retain_action_outcome_v1( out.extend_from_slice(&obstruction.invocation_id.as_hash()); out.extend_from_slice(&obstruction.evaluation_basis_id.as_hash()); } - EchoOperationActionOutcomeV1::RejectedFootprintConflict { - installed_operation_id, - invocation_id, - preparation_id, - blocked_by, - } => { + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { out.push(3); - out.extend_from_slice(&installed_operation_id.as_hash()); - out.extend_from_slice(&invocation_id.as_hash()); - out.extend_from_slice(&preparation_id.as_hash()); - let blocker_count = u64::try_from(blocked_by.len()) + out.extend_from_slice(&conflict.package_id.as_hash()); + out.extend_from_slice(&conflict.installed_operation_id.as_hash()); + out.extend_from_slice(&conflict.invocation_admission_policy_id); + out.extend_from_slice( + &conflict + .invocation_admission_maximum_budget + .steps + .to_le_bytes(), + ); + out.extend_from_slice( + &conflict + .invocation_admission_maximum_budget + .read_bytes + .to_le_bytes(), + ); + out.extend_from_slice( + &conflict + .invocation_admission_maximum_budget + .write_bytes + .to_le_bytes(), + ); + out.extend_from_slice(&conflict.invocation_admission_id.as_hash()); + out.extend_from_slice(&conflict.invocation_id.as_hash()); + out.extend_from_slice(&conflict.evaluation_basis_id.as_hash()); + out.extend_from_slice(&conflict.private_evaluation_id.as_hash()); + out.extend_from_slice(&conflict.prepared_patch_digest); + out.extend_from_slice(&conflict.prepared_result_id.as_hash()); + out.extend_from_slice(&conflict.actual_footprint_digest); + out.extend_from_slice(&conflict.preparation_id.as_hash()); + let blocker_count = u64::try_from(conflict.blocked_by.len()) .map_err(|_| invalid_structure("Action blocker count is not representable"))?; out.extend_from_slice(&blocker_count.to_le_bytes()); - for blocker in blocked_by { + for blocker in &conflict.blocked_by { out.extend_from_slice(&blocker.to_le_bytes()); } } @@ -2851,10 +3084,28 @@ pub(crate) fn recover_action_outcome_v1( }) } 3 => { + let package_id = + EchoOperationPackageIdV1(read_action_outcome_hash(bytes, &mut offset)?); let installed_operation_id = InstalledEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let invocation_admission_policy_id = read_action_outcome_hash(bytes, &mut offset)?; + let invocation_admission_maximum_budget = EchoOperationBudgetV1 { + steps: read_action_outcome_u64(bytes, &mut offset)?, + read_bytes: read_action_outcome_u64(bytes, &mut offset)?, + write_bytes: read_action_outcome_u64(bytes, &mut offset)?, + }; + let invocation_admission_id = + EchoOperationInvocationAdmissionIdV1(read_action_outcome_hash(bytes, &mut offset)?); let invocation_id = EchoOperationInvocationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let evaluation_basis_id = + EchoOperationEvaluationBasisIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let private_evaluation_id = + EchoOperationPrivateEvaluationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let prepared_patch_digest = read_action_outcome_hash(bytes, &mut offset)?; + let prepared_result_id = + EchoOperationResultIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let actual_footprint_digest = read_action_outcome_hash(bytes, &mut offset)?; let preparation_id = PreparedEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); let blocker_count = read_action_outcome_u64(bytes, &mut offset)?; @@ -2879,12 +3130,23 @@ pub(crate) fn recover_action_outcome_v1( offset = end; blocked_by.push(u32::from_le_bytes(raw)); } - EchoOperationActionOutcomeV1::RejectedFootprintConflict { - installed_operation_id, - invocation_id, - preparation_id, - blocked_by, - } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(Box::new( + EchoOperationFootprintConflictV1 { + package_id, + installed_operation_id, + invocation_admission_policy_id, + invocation_admission_maximum_budget, + invocation_admission_id, + invocation_id, + evaluation_basis_id, + private_evaluation_id, + prepared_patch_digest, + prepared_result_id, + actual_footprint_digest, + preparation_id, + blocked_by, + }, + )) } _ => { return Err(invalid_structure( @@ -3017,6 +3279,14 @@ impl PreparedEchoOperationV1 { self.invocation_admission_id } + pub(crate) const fn invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission_policy_id + } + + pub(crate) const fn invocation_admission_maximum_budget(&self) -> EchoOperationBudgetV1 { + self.invocation_admission_maximum_budget + } + /// Returns the declared-footprint identity bound by evaluation. #[must_use] pub const fn declared_footprint_digest(&self) -> Hash { @@ -3059,6 +3329,10 @@ impl PreparedEchoOperationV1 { self.result_id } + pub(crate) fn prepared_patch_digest(&self) -> Hash { + self.patch.digest() + } + pub(crate) const fn package_id(&self) -> EchoOperationPackageIdV1 { self.installed.package_id } @@ -3487,6 +3761,16 @@ impl EchoOperationReceiptV1 { self.invocation_admission_id } + pub(crate) const fn retained_invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission_policy_id + } + + pub(crate) const fn retained_invocation_admission_maximum_budget( + &self, + ) -> EchoOperationBudgetV1 { + self.invocation_admission_maximum_budget + } + /// Returns the bounded private-evaluation identity. #[must_use] pub const fn private_evaluation_id(&self) -> EchoOperationPrivateEvaluationIdV1 { @@ -4183,12 +4467,7 @@ pub(crate) struct EchoOperationActionBatchCommitMaterialV1 { enum SchedulerEchoOperationDecisionV1 { Applied(Box), Obstructed(EchoOperationObstructionV1), - RejectedFootprintConflict { - installed_operation_id: InstalledEchoOperationIdV1, - invocation_id: EchoOperationInvocationIdV1, - preparation_id: PreparedEchoOperationIdV1, - blocked_by: Vec, - }, + RejectedFootprintConflict(Box), } #[derive(Clone, Copy, Debug)] @@ -4296,12 +4575,25 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( blocked_by.push(blockers.clone()); decisions.push(( candidate.submission_id, - SchedulerEchoOperationDecisionV1::RejectedFootprintConflict { - installed_operation_id: prepared.installed_operation_id(), - invocation_id: prepared.invocation_id(), - preparation_id: prepared.preparation_id(), - blocked_by: blockers, - }, + SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(Box::new( + EchoOperationFootprintConflictV1 { + package_id: prepared.package_id(), + installed_operation_id: prepared.installed_operation_id(), + invocation_admission_policy_id: prepared + .invocation_admission_policy_id(), + invocation_admission_maximum_budget: prepared + .invocation_admission_maximum_budget(), + invocation_admission_id: prepared.invocation_admission_id(), + invocation_id: prepared.invocation_id(), + evaluation_basis_id: prepared.evaluation_basis().identity(), + private_evaluation_id: prepared.private_evaluation_id(), + prepared_patch_digest: prepared.prepared_patch_digest(), + prepared_result_id: prepared.result_id(), + actual_footprint_digest: prepared.actual_footprint_digest(), + preparation_id: prepared.preparation_id(), + blocked_by: blockers, + }, + )), )); } } @@ -4324,7 +4616,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( .filter_map(|(_, decision)| match decision { SchedulerEchoOperationDecisionV1::Applied(prepared) => Some(prepared.as_ref()), SchedulerEchoOperationDecisionV1::Obstructed(_) - | SchedulerEchoOperationDecisionV1::RejectedFootprintConflict { .. } => None, + | SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(_) => None, }) .collect::>(); let rewrites_digest = action_batch_rewrites_digest(&applied); @@ -4384,17 +4676,9 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( SchedulerEchoOperationDecisionV1::Obstructed(obstruction) => { EchoOperationActionOutcomeV1::Obstructed(obstruction) } - SchedulerEchoOperationDecisionV1::RejectedFootprintConflict { - installed_operation_id, - invocation_id, - preparation_id, - blocked_by, - } => EchoOperationActionOutcomeV1::RejectedFootprintConflict { - installed_operation_id, - invocation_id, - preparation_id, - blocked_by, - }, + SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(conflict) => { + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) + } }; (submission_id, outcome) }) diff --git a/crates/warp-core/src/lib.rs b/crates/warp-core/src/lib.rs index 45ce50ea..25156c50 100644 --- a/crates/warp-core/src/lib.rs +++ b/crates/warp-core/src/lib.rs @@ -247,17 +247,18 @@ pub use echo_operation::{ EchoOperationApplicationBasisV1, EchoOperationArtifactErrorKindV1, EchoOperationArtifactErrorV1, EchoOperationBudgetV1, EchoOperationCommitErrorV1, EchoOperationEvaluationBasisIdV1, EchoOperationEvaluationBasisV1, - EchoOperationExecutionEvidenceV1, EchoOperationFootprintContractV1, - EchoOperationInstallationErrorKindV1, EchoOperationInstallationErrorV1, - EchoOperationInvocationAdmissionErrorKindV1, EchoOperationInvocationAdmissionErrorV1, - EchoOperationInvocationAdmissionIdV1, EchoOperationInvocationAdmissionPolicyV1, - EchoOperationInvocationIdV1, EchoOperationInvocationV1, EchoOperationObstructionIdV1, - EchoOperationObstructionKindV1, EchoOperationObstructionV1, EchoOperationPackageAdmissionIdV1, - EchoOperationPackageIdV1, EchoOperationPreparationV1, EchoOperationPrivateEvaluationIdV1, - EchoOperationProgramIdV1, EchoOperationProgramV1, EchoOperationReceiptV1, - EchoOperationResultIdV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, - ExecutableOperationPackageV1, InstalledEchoOperationIdV1, InstalledEchoOperationV1, - PreparedEchoOperationIdV1, PreparedEchoOperationV1, + EchoOperationExecutionEvidenceV1, EchoOperationFootprintConflictV1, + EchoOperationFootprintContractV1, EchoOperationInstallationErrorKindV1, + EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorKindV1, + EchoOperationInvocationAdmissionErrorV1, EchoOperationInvocationAdmissionIdV1, + EchoOperationInvocationAdmissionPolicyV1, EchoOperationInvocationIdV1, + EchoOperationInvocationV1, EchoOperationObstructionIdV1, EchoOperationObstructionKindV1, + EchoOperationObstructionV1, EchoOperationPackageAdmissionIdV1, EchoOperationPackageIdV1, + EchoOperationPreparationV1, EchoOperationPrivateEvaluationIdV1, EchoOperationProgramIdV1, + EchoOperationProgramV1, EchoOperationReceiptV1, EchoOperationResultIdV1, + EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, ExecutableOperationPackageV1, + InstalledEchoOperationIdV1, InstalledEchoOperationV1, PreparedEchoOperationIdV1, + PreparedEchoOperationV1, }; pub use edict_target_ir::{ accept_edict_echo_target_ir, execute_accepted_edict_echo_target_ir, AcceptedEdictEchoTargetIr, diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index eba4aaee..e271b3bc 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -42,13 +42,14 @@ use crate::{ }, contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ - action_application_basis_matches_state_v1, - action_batch_composition_digest_from_receipts_v1, admit_action_invocation_v1, - admit_invocation_v1, admit_package_v1, commit_prepared_to_state, - decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, - inspect_action_invocation_v1, install_recovered_v1, installed_from_admitted, - not_committed_basis_changed, not_committed_evaluation_authority_mismatch, - not_committed_installation_unavailable, operation_descent_stack, prepare_operation_v1, + action_admission_evidence_matches_v1, action_application_basis_matches_state_v1, + action_batch_composition_digest_from_receipts_v1, action_preparation_identity_matches_v1, + admit_action_invocation_v1, admit_invocation_v1, admit_package_v1, + commit_prepared_to_state, decode_invocation_route_v1, + echo_operation_action_invocation_bytes_v1, inspect_action_invocation_v1, + install_recovered_v1, installed_from_admitted, not_committed_basis_changed, + not_committed_evaluation_authority_mismatch, not_committed_installation_unavailable, + operation_descent_stack, prepare_operation_v1, reconstruct_action_preparation_v1, recover_action_outcome_v1, recover_committed_execution_receipt_v1, recover_installation_v1, retain_action_outcome_v1, retain_committed_execution_v1, retain_installation_v1, validate_receipt_installation_v1, EchoOperationEvaluationAuthorityV1, @@ -498,6 +499,48 @@ impl TrustedRuntimeWalRecovery { .remove(&submission_id); } + /// Replaces one rejected preparation identity for adversarial tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_conflict_preparation_for_test( + &mut self, + submission_id: Hash, + digest: Hash, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(candidate, _, _)| *candidate == submission_id) + { + outcome.replace_conflict_preparation_id_for_test(digest); + } + } + + /// Replaces one retained conflict's blockers for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_conflict_blockers_for_test( + &mut self, + submission_id: Hash, + replacement: Vec, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(retained_submission_id, _, _)| *retained_submission_id == submission_id) + { + outcome.replace_conflict_blockers_for_test(replacement); + } + } + + /// Re-runs activation-time Action parent-state checks for adversarial tests. + #[cfg(any(test, feature = "host_test"))] + pub fn validate_echo_operation_parent_states_for_test( + &self, + runtime: &WorldlineRuntime, + provenance: &ProvenanceService, + ) -> Result<(), TrustedRuntimeWalError> { + validate_recovered_echo_operation_parent_states(runtime, provenance, self) + } + /// Recomputes the certificate's canonical index root from recovered evidence. /// /// # Errors @@ -3817,7 +3860,22 @@ fn validate_recovered_echo_operation_parent_states( .iter() .map(|correlation| (correlation.submission_id, correlation)) .collect::>(); - for (submission_id, _, outcome) in &recovery.echo_operation_action_outcomes { + let outcomes = recovery + .echo_operation_action_outcomes + .iter() + .map(|(submission_id, _, outcome)| (*submission_id, outcome)) + .collect::>(); + let mut reconstructed_preparations = BTreeMap::new(); + let mut action_tick_members = BTreeMap::< + ( + crate::WriterHeadKey, + crate::WorldlineTick, + crate::GlobalTick, + Hash, + ), + Vec<(Hash, Hash)>, + >::new(); + for (submission_id, ingress_id, outcome) in &recovery.echo_operation_action_outcomes { let submission = submissions.get(submission_id).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action outcome has no retained submission", @@ -3883,6 +3941,136 @@ fn validate_recovered_echo_operation_parent_states( detail: "Action basis posture disagrees with the scheduler Tick parent", }); } + let transition = entries.get(&(worldline_id, tick_before)).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no recovered scheduler transition", + }, + )?; + let policy_id = transition + .patch + .as_ref() + .map(crate::WorldlineTickPatchV1::policy_id) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome scheduler transition has no retained patch", + })?; + let reconstructed = match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + let prepared = reconstruct_action_preparation_v1( + installed, + invocation_bytes, + receipt.retained_invocation_admission_maximum_budget(), + receipt.retained_invocation_admission_policy_id(), + receipt.invocation_admission_id(), + &basis_state, + policy_id, + ) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "committed Action preparation cannot be reconstructed", + })?; + if prepared.package_id() != receipt.package_id() + || prepared.installed_operation_id() != receipt.installed_operation_id() + || prepared.invocation_id() != receipt.invocation_id() + || prepared.evaluation_basis().identity() != receipt.evaluation_basis_id() + || prepared.private_evaluation_id() != receipt.private_evaluation_id() + || prepared.prepared_patch_digest() != receipt.prepared_patch_digest() + || prepared.result_id() != receipt.prepared_result_id() + || prepared.actual_footprint_digest() != receipt.actual_footprint_digest() + || prepared.preparation_id() != receipt.preparation_id() + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: + "committed Action evidence disagrees with reconstructed preparation", + }); + } + Some(prepared) + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + let prepared = reconstruct_action_preparation_v1( + installed, + invocation_bytes, + conflict.invocation_admission_maximum_budget, + conflict.invocation_admission_policy_id, + conflict.invocation_admission_id, + &basis_state, + policy_id, + ) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflicting Action preparation cannot be reconstructed", + })?; + if prepared.package_id() != conflict.package_id + || prepared.installed_operation_id() != conflict.installed_operation_id + || prepared.invocation_id() != conflict.invocation_id + || prepared.evaluation_basis().identity() != conflict.evaluation_basis_id + || prepared.private_evaluation_id() != conflict.private_evaluation_id + || prepared.prepared_patch_digest() != conflict.prepared_patch_digest + || prepared.result_id() != conflict.prepared_result_id + || prepared.actual_footprint_digest() != conflict.actual_footprint_digest + || prepared.preparation_id() != conflict.preparation_id + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflict evidence disagrees with reconstructed preparation", + }); + } + Some(prepared) + } + EchoOperationActionOutcomeV1::Obstructed(_) => None, + }; + if let Some(prepared) = reconstructed { + reconstructed_preparations.insert(*submission_id, prepared); + } + action_tick_members + .entry(( + correlation.head_key, + correlation.worldline_tick_after, + correlation.commit_global_tick, + correlation.commit_hash, + )) + .or_default() + .push((*ingress_id, *submission_id)); + } + for (_, mut members) in action_tick_members { + members.sort_by_key(|(ingress_id, _)| *ingress_id); + for (index, (_, submission_id)) in members.iter().enumerate() { + let Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) = + outcomes.get(submission_id) + else { + continue; + }; + let candidate = reconstructed_preparations.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflicting Action has no reconstructed preparation", + }, + )?; + let mut expected_blockers = Vec::new(); + for (earlier_index, (_, earlier_submission_id)) in members[..index].iter().enumerate() { + if !matches!( + outcomes.get(earlier_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + ) { + continue; + } + let earlier = reconstructed_preparations + .get(earlier_submission_id) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "committed Action has no reconstructed preparation", + })?; + if crate::engine_impl::footprints_conflict( + candidate.actual_footprint(), + earlier.actual_footprint(), + ) { + expected_blockers.push(u32::try_from(earlier_index).map_err(|_| { + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has too many members to index", + } + })?); + } + } + if expected_blockers != conflict.blocked_by { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflicting Action blockers disagree with reconstructed footprints", + }); + } + } } Ok(()) } @@ -4325,7 +4513,9 @@ fn validate_recovered_echo_operation_action_outcomes( EchoOperationActionOutcomeV1::Obstructed(obstruction) => { Some(obstruction.invocation_admission_id()) } - EchoOperationActionOutcomeV1::RejectedFootprintConflict { .. } => None, + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + Some(conflict.invocation_admission_id) + } }; if correlation.ticket_digest != correlation.causal_receipt_ref.ticket_digest || invocation_admission_id.is_some_and(|admission_id| { @@ -4366,16 +4556,29 @@ fn validate_recovered_echo_operation_action_outcomes( return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } } - EchoOperationActionOutcomeV1::RejectedFootprintConflict { - installed_operation_id, - invocation_id, - blocked_by, - .. - } => { - if *installed_operation_id != installed.installed_operation_id() - || blocked_by.is_empty() - || blocked_by.windows(2).any(|pair| pair[0] >= pair[1]) - || *invocation_id != invocation.invocation_id + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + if conflict.package_id != invocation.package_id + || conflict.installed_operation_id != installed.installed_operation_id() + || !action_admission_evidence_matches_v1( + installed, + invocation_bytes, + conflict.invocation_admission_maximum_budget, + conflict.invocation_admission_policy_id, + conflict.invocation_admission_id, + ) + || conflict.evaluation_basis_id != invocation.evaluation_basis.identity() + || !action_preparation_identity_matches_v1( + conflict.private_evaluation_id, + conflict.prepared_patch_digest, + conflict.prepared_result_id, + conflict.preparation_id, + ) + || conflict.blocked_by.is_empty() + || conflict + .blocked_by + .windows(2) + .any(|pair| pair[0] >= pair[1]) + || conflict.invocation_id != invocation.invocation_id { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } @@ -4408,7 +4611,7 @@ fn validate_recovered_echo_operation_action_outcomes( .filter_map(|(_, outcome, _, _)| match outcome { EchoOperationActionOutcomeV1::Committed(receipt) => Some(receipt.as_ref()), EchoOperationActionOutcomeV1::Obstructed(_) - | EchoOperationActionOutcomeV1::RejectedFootprintConflict { .. } => None, + | EchoOperationActionOutcomeV1::RejectedFootprintConflict(_) => None, }) .collect::>(); if !committed_receipts.is_empty() { @@ -4464,15 +4667,11 @@ fn validate_recovered_echo_operation_action_outcomes( return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } } - EchoOperationActionOutcomeV1::RejectedFootprintConflict { - installed_operation_id, - blocked_by, - .. - } => { + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { if tick_entry.disposition != TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) - || tick_entry.rule_id != installed_operation_id.as_hash() - || blockers != blocked_by + || tick_entry.rule_id != conflict.installed_operation_id.as_hash() + || blockers != conflict.blocked_by { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } @@ -4487,7 +4686,7 @@ fn wal_tick_decision_for_action_outcome(outcome: &EchoOperationActionOutcomeV1) match outcome { EchoOperationActionOutcomeV1::Committed(_) => WalTickDecision::Applied, EchoOperationActionOutcomeV1::Obstructed(_) => WalTickDecision::Obstructed, - EchoOperationActionOutcomeV1::RejectedFootprintConflict { .. } => { + EchoOperationActionOutcomeV1::RejectedFootprintConflict(_) => { WalTickDecision::RejectedFootprintConflict } } diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index a7b29928..1afd12c1 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3540,6 +3540,88 @@ fn recovery_separates_identical_commits_on_distinct_worldlines() { } } +#[test] +fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { + let wal_dir = TempWalDir::new(); + let conflict_submission_id; + + { + let (mut host, head_key, node) = fixture_host(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the footprint-conflict fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let mut submission_ids = Vec::new(); + for replacement in [b"after-a".as_slice(), b"after-b".as_slice()] { + let invocation = + action_invocation(&host, &installed, head_key, node, b"before", replacement); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the conflicting invocation becomes an Action"); + submission_ids.push( + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the conflicting Action is durable") + .submission_id, + ); + } + + let steps = host + .tick_once() + .expect("the scheduler lawfully classifies the footprint conflict"); + assert_eq!(steps.len(), 1); + conflict_submission_id = submission_ids + .into_iter() + .find(|submission_id| { + matches!( + host.echo_operation_action_outcome_v1(submission_id), + Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(_)) + ) + }) + .expect("exactly one Action is rejected by the earlier applied footprint"); + + let mut adversarial = host + .runtime_wal() + .expect("the conflict WAL remains enabled") + .recover_read_only() + .expect("the honest conflict batch recovers"); + adversarial.replace_echo_operation_action_conflict_preparation_for_test( + conflict_submission_id, + digest("forged-conflict-preparation"), + ); + assert!(matches!( + adversarial.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut false_conflict = host + .runtime_wal() + .expect("the conflict WAL remains enabled") + .recover_read_only() + .expect("the honest conflict batch recovers again"); + false_conflict.replace_echo_operation_action_conflict_blockers_for_test( + conflict_submission_id, + Vec::new(), + ); + assert!(matches!( + false_conflict + .validate_echo_operation_parent_states_for_test(host.runtime(), host.provenance()), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); + } + + let (mut recovered, _, _) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("fresh-host recovery reconstructs the honest footprint conflict"); + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&conflict_submission_id), + Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(_)) + )); +} + #[test] fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { let wal_dir = TempWalDir::new(); From fcc964cf14b45195a3472e41b7bdc55b6481a5f5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:10:01 -0700 Subject: [PATCH 24/64] Fix: reject Action outcomes in legacy Tick framing --- crates/warp-core/src/trusted_runtime_host.rs | 89 +++++++++++++++++--- 1 file changed, 75 insertions(+), 14 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index e271b3bc..b91b86fa 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -4873,22 +4873,14 @@ fn tick_record_batch_from_transaction( )); } index += 2; - let action_outcome = if transaction.frames.get(index).is_some_and(|frame| { + if transaction.frames.get(index).is_some_and(|frame| { frame.header.record_kind == WalRecordKind::ExecutableOperationActionOutcomeRecorded }) { - let frame = &transaction.frames[index]; - index += 1; - let recovered = recover_action_outcome_v1(&frame.payload.canonical_bytes)?; - if recovered.0 != receipt.receipt_ref.submission_id { - return Err(decode_trusted_runtime_wal_payload( - WalDecodeError::InvalidEmbeddedFrame, - )); - } - Some(recovered) - } else { - None - }; - records.push((receipt, correlation, action_outcome)); + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + records.push((receipt, correlation, None)); } } let action_outcome_count = records @@ -6723,6 +6715,75 @@ mod tests { } } + #[test] + fn runtime_wal_legacy_tick_rejects_action_outcome_frame() { + let wal = TrustedRuntimeWal::new_in_memory().expect("test WAL should initialize"); + let receipt = TickReceiptRecord { + receipt_ref: CausalTickReceiptRef { + worldline_id: WorldlineId::from_bytes([40; 32]), + worldline_tick_after: WorldlineTick::from_raw(1), + commit_global_tick: GlobalTick::from_raw(1), + commit_hash: [41; 32], + submission_id: [42; 32], + ticket_digest: [43; 32], + receipt_content_digest: [44; 32], + }, + decision: WalTickDecision::Obstructed, + }; + let correlation = WalReceiptCorrelationRecord { + receipt_ref: receipt.receipt_ref, + causal_parent_receipts: Vec::new(), + }; + let mut action_outcome = b"EOACT002".to_vec(); + action_outcome.extend_from_slice(&receipt.receipt_ref.submission_id); + action_outcome.extend_from_slice(&[45; 32]); + action_outcome.extend_from_slice(&[2, 1]); + for byte in 46..=50 { + action_outcome.extend_from_slice(&[byte; 32]); + } + + let mut builder = wal.builder( + WalTransactionKind::SchedulerTick, + WalAppendAuthority::TrustedScheduler, + WalTransactionId::from_hash([51; 32]), + ); + builder + .push_record( + WalRecordKind::TickReceiptRecorded, + receipt.to_payload_bytes(), + ) + .expect("fixture legacy receipt must append"); + builder + .push_record( + WalRecordKind::ReceiptCorrelationRecorded, + correlation.to_payload_bytes(), + ) + .expect("fixture legacy correlation must append"); + builder + .push_record( + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + action_outcome, + ) + .expect("adversarial Action outcome must append structurally"); + builder + .push_record(WalRecordKind::RuntimeStateDeltaRecorded, [52; 32].to_vec()) + .expect("fixture state delta must append"); + let transaction = builder + .commit(Vec::new()) + .expect("adversarial legacy transaction must commit structurally"); + let recovered = crate::causal_wal::WalRecoveredTransaction { + commit: transaction.commit, + frames: transaction.frames, + }; + + assert!(matches!( + tick_record_batch_from_transaction(&recovered), + Err(TrustedRuntimeWalError::Recovery(WalRecoveryError::Index( + WalRecoveryIndexError::Decode(WalDecodeError::InvalidEmbeddedFrame) + ))) + )); + } + #[test] fn recovered_correlation_rejects_parents_not_bound_by_envelope() { let head_key = test_head_key(); From 0f0bd9e1624daf35116afd75e5ad6a9fa6e71020 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:16:03 -0700 Subject: [PATCH 25/64] Fix: bound executable Action Tick composition --- crates/warp-core/src/coordinator.rs | 1 + crates/warp-core/src/echo_operation.rs | 82 ++++++++++++++++--- crates/warp-core/src/head_inbox.rs | 17 +++- .../executable_operation_pipeline_tests.rs | 61 ++++++++++++++ 4 files changed, 147 insertions(+), 14 deletions(-) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index e1e990fb..219ea400 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -4225,6 +4225,7 @@ impl SchedulerCoordinator { let admitted = if operation_actions.is_some() { inbox.admit_partitioned( crate::echo_operation::echo_operation_action_intent_kind_v1(), + crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1, ) } else { inbox.admit() diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index b552f875..49c04795 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -85,6 +85,11 @@ const ACTION_BATCH_PLAN_DOMAIN: &[u8] = b"echo:operation-action-batch-plan:v1\0" const ACTION_BATCH_REWRITES_DOMAIN: &[u8] = b"echo:operation-action-batch-rewrites:v1\0"; const ACTION_BATCH_COMPOSITION_DOMAIN: &[u8] = b"echo:operation-action-batch-composition:v1\0"; const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT002"; +pub(crate) const ACTION_BATCH_CANDIDATE_LIMIT_V1: usize = 64; +const ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1: usize = + ACTION_BATCH_CANDIDATE_LIMIT_V1 * (ACTION_BATCH_CANDIDATE_LIMIT_V1 - 1) / 2; +const ACTION_BATCH_BLOCKER_EVIDENCE_LIMIT_V1: usize = ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1; +const ACTION_BATCH_OPERATION_LIMIT_V1: usize = ACTION_BATCH_CANDIDATE_LIMIT_V1 * 2; const INVOCATION_BYTES_DIGEST_DOMAIN: &[u8] = b"echo:operation-invocation-bytes:v1\0"; const BASIS_ID_DOMAIN: &[u8] = b"echo:operation-evaluation-basis:v1\0"; const PACKAGE_ADMISSION_ID_DOMAIN: &[u8] = b"echo:operation-package-admission:v1\0"; @@ -4470,6 +4475,19 @@ enum SchedulerEchoOperationDecisionV1 { RejectedFootprintConflict(Box), } +fn scheduler_composition_budget_obstruction_v1( + prepared: &PreparedEchoOperationV1, +) -> EchoOperationObstructionV1 { + EchoOperationObstructionV1 { + kind: EchoOperationObstructionKindV1::BudgetExceeded, + package_id: prepared.package_id(), + installed_operation_id: prepared.installed_operation_id(), + invocation_admission_id: prepared.invocation_admission_id(), + invocation_id: prepared.invocation_id(), + evaluation_basis_id: prepared.evaluation_basis().identity(), + } +} + #[derive(Clone, Copy, Debug)] struct EchoOperationTerminalMaterialV1 { posture: EchoOperationTerminalPostureV1, @@ -4487,6 +4505,9 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( commit_global_tick: GlobalTick, policy_id: u32, ) -> Result { + if candidates.len() > ACTION_BATCH_CANDIDATE_LIMIT_V1 { + return Err(EchoOperationCommitErrorV1::TooManyCandidates); + } // This order is part of the receipt-correlation contract: coordinator // outcome lookup maps the same Tick's ingress-sorted correlations onto // these receipt entries by position. @@ -4513,6 +4534,8 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( let mut out_slots = Vec::new(); let mut ops = Vec::new(); let mut ordered_rule_ids = Vec::with_capacity(candidates.len()); + let mut footprint_comparisons = 0_usize; + let mut blocker_evidence_count = 0_usize; for (entry_index, candidate) in candidates.into_iter().enumerate() { let entry_index_u32 = u32::try_from(entry_index) @@ -4536,16 +4559,54 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( )); } EchoOperationPreparationV1::Prepared(prepared) => { - let blockers = accepted_footprints - .iter() - .filter_map(|(accepted_index, footprint)| { - crate::engine_impl::footprints_conflict( - prepared.actual_footprint(), - footprint, - ) - .then_some(*accepted_index) - }) - .collect::>(); + let comparison_cost = accepted_footprints.len(); + let comparison_budget_exceeded = footprint_comparisons + .checked_add(comparison_cost) + .is_none_or(|total| total > ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1); + let blockers = if comparison_budget_exceeded { + Vec::new() + } else { + footprint_comparisons += comparison_cost; + accepted_footprints + .iter() + .filter_map(|(accepted_index, footprint)| { + crate::engine_impl::footprints_conflict( + prepared.actual_footprint(), + footprint, + ) + .then_some(*accepted_index) + }) + .collect::>() + }; + let blocker_budget_exceeded = blocker_evidence_count + .checked_add(blockers.len()) + .is_none_or(|total| total > ACTION_BATCH_BLOCKER_EVIDENCE_LIMIT_V1); + let operation_budget_exceeded = blockers.is_empty() + && ops + .len() + .checked_add(prepared.patch().ops().len()) + .is_none_or(|total| total > ACTION_BATCH_OPERATION_LIMIT_V1); + if comparison_budget_exceeded + || blocker_budget_exceeded + || operation_budget_exceeded + { + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ), + }); + blocked_by.push(Vec::new()); + decisions.push(( + candidate.submission_id, + SchedulerEchoOperationDecisionV1::Obstructed( + scheduler_composition_budget_obstruction_v1(&prepared), + ), + )); + continue; + } if blockers.is_empty() { entries.push(TickReceiptEntry { rule_id: candidate.rule_id, @@ -4564,6 +4625,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( SchedulerEchoOperationDecisionV1::Applied(prepared), )); } else { + blocker_evidence_count += blockers.len(); entries.push(TickReceiptEntry { rule_id: candidate.rule_id, scope_hash, diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index 158dc898..950c0f0b 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -799,7 +799,11 @@ impl HeadInbox { /// category remain pending for a later Tick. Existing per-Tick limits still /// bound the selected category. #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] - pub(crate) fn admit_partitioned(&mut self, partition_kind: IntentKind) -> Vec { + pub(crate) fn admit_partitioned( + &mut self, + partition_kind: IntentKind, + partition_limit: usize, + ) -> Vec { let Some(first) = self.pending.first_key_value().map(|(_, envelope)| envelope) else { return Vec::new(); }; @@ -808,10 +812,15 @@ impl HeadInbox { IngressPayload::LocalIntent { intent_kind, .. } if *intent_kind == partition_kind ); - let limit = match self.policy { + let policy_limit = match self.policy { InboxPolicy::Budgeted { max_per_tick } => max_per_tick as usize, InboxPolicy::AcceptAll | InboxPolicy::KindFilter(_) => usize::MAX, }; + let limit = if selected_partition { + policy_limit.min(partition_limit) + } else { + policy_limit + }; if limit == 0 { return Vec::new(); } @@ -975,7 +984,7 @@ mod tests { assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); } - let first_batch = inbox.admit_partitioned(partition_kind); + let first_batch = inbox.admit_partitioned(partition_kind, 2); assert_eq!(first_batch.len(), 2); assert!(first_batch.iter().all(|envelope| { matches!( @@ -986,7 +995,7 @@ mod tests { })); assert_eq!(inbox.pending_count(), 2); - let second_batch = inbox.admit_partitioned(partition_kind); + let second_batch = inbox.admit_partitioned(partition_kind, 2); assert_eq!(second_batch.len(), 2); assert!(second_batch.iter().all(|envelope| { matches!( diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 1afd12c1..74f96ce1 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3006,6 +3006,67 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { ); } +#[test] +fn scheduler_candidate_limit_leaves_excess_action_pending() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the scheduler budget fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let mut submission_ids = Vec::new(); + for index in 0..65 { + let replacement = format!("bounded-after-{index}"); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + replacement.as_bytes(), + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the bounded invocation becomes one canonical Action"); + submission_ids.push( + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the bounded Action is durable") + .submission_id, + ); + } + assert_eq!(host.runtime().pending_witnessed_submission_count(), 65); + + let first_tick = host + .tick_once() + .expect("the first scheduler Tick respects its Action candidate limit"); + assert_eq!(first_tick.len(), 1); + assert_eq!(first_tick[0].admitted_count, 64); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); + assert_eq!( + submission_ids + .iter() + .filter(|submission_id| { + host.echo_operation_action_outcome_v1(submission_id) + .is_some() + }) + .count(), + 64 + ); + + let second_tick = host + .tick_once() + .expect("the excess Action remains available to a later scheduler Tick"); + assert_eq!(second_tick.len(), 1); + assert_eq!(second_tick[0].admitted_count, 1); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 0); + assert!(submission_ids.iter().all(|submission_id| host + .echo_operation_action_outcome_v1(submission_id) + .is_some())); +} + #[test] fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { let wal_dir = TempWalDir::new(); From 5402ef0adfd514e2ada62f6981f67034f97b5cea Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:23:30 -0700 Subject: [PATCH 26/64] Fix: index pending executable Actions for admission --- crates/warp-core/src/trusted_runtime_host.rs | 180 +++++++++++++++--- .../executable_operation_pipeline_tests.rs | 37 ++++ 2 files changed, 187 insertions(+), 30 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index b91b86fa..4968303e 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -713,8 +713,31 @@ pub struct TrustedRuntimeHost { echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1, echo_operation_action_admission_policy: Option, echo_operation_action_outcomes: BTreeMap, + pending_echo_operation_actions: BTreeSet, + admitted_echo_operation_actions: BTreeMap, echo_operation_action_admission_obstructions: BTreeMap, + #[cfg(any(test, feature = "host_test"))] + echo_operation_action_admission_attempts: BTreeMap, +} + +fn pending_echo_operation_action_ids_v1( + runtime: &WorldlineRuntime, + decided: &BTreeMap, +) -> BTreeSet { + runtime + .pending_witnessed_submissions() + .filter_map(|submission| { + let submission_id = submission.submission_id; + (!decided.contains_key(&submission_id) + && runtime + .witnessed_submission_envelope(&submission_id) + .is_some_and(|envelope| { + echo_operation_action_invocation_bytes_v1(envelope).is_some() + })) + .then_some(submission_id) + }) + .collect() } impl TrustedRuntimeHost { @@ -726,6 +749,8 @@ impl TrustedRuntimeHost { /// Returns a provenance error if any runtime worldline cannot be registered. pub fn new(runtime: WorldlineRuntime, engine: Engine) -> Result { let provenance = provenance_from_runtime(&runtime)?; + let pending_echo_operation_actions = + pending_echo_operation_action_ids_v1(&runtime, &BTreeMap::new()); Ok(Self { runtime, provenance, @@ -735,7 +760,11 @@ impl TrustedRuntimeHost { echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), echo_operation_action_admission_policy: None, echo_operation_action_outcomes: BTreeMap::new(), + pending_echo_operation_actions, + admitted_echo_operation_actions: BTreeMap::new(), echo_operation_action_admission_obstructions: BTreeMap::new(), + #[cfg(any(test, feature = "host_test"))] + echo_operation_action_admission_attempts: BTreeMap::new(), }) } @@ -746,6 +775,8 @@ impl TrustedRuntimeHost { provenance: ProvenanceService, engine: Engine, ) -> Self { + let pending_echo_operation_actions = + pending_echo_operation_action_ids_v1(&runtime, &BTreeMap::new()); Self { runtime, provenance, @@ -755,7 +786,11 @@ impl TrustedRuntimeHost { echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), echo_operation_action_admission_policy: None, echo_operation_action_outcomes: BTreeMap::new(), + pending_echo_operation_actions, + admitted_echo_operation_actions: BTreeMap::new(), echo_operation_action_admission_obstructions: BTreeMap::new(), + #[cfg(any(test, feature = "host_test"))] + echo_operation_action_admission_attempts: BTreeMap::new(), } } @@ -790,7 +825,7 @@ impl TrustedRuntimeHost { policy: EchoOperationInvocationAdmissionPolicyV1, ) { self.echo_operation_action_admission_policy = Some(policy); - self.echo_operation_action_admission_obstructions.clear(); + self.requeue_obstructed_echo_operation_actions_v1(); } /// Returns the typed scheduler-owned outcome for one executable-operation @@ -815,6 +850,17 @@ impl TrustedRuntimeHost { .copied() } + /// Returns how often runtime-owned admission evaluated one Action in + /// scheduler tests. + #[cfg(any(test, feature = "host_test"))] + #[must_use] + pub fn echo_operation_action_admission_attempts_for_test(&self, submission_id: &Hash) -> u64 { + self.echo_operation_action_admission_attempts + .get(submission_id) + .copied() + .unwrap_or(0) + } + /// Admits exact executable-operation package bytes under independent policy. /// /// A successful result is an opaque admission token. It does not install @@ -860,7 +906,7 @@ impl TrustedRuntimeHost { } self.engine .restore_recovered_echo_operation_packages_v1(core::slice::from_ref(&installed))?; - self.echo_operation_action_admission_obstructions.clear(); + self.requeue_obstructed_echo_operation_actions_v1(); Ok(installed) } @@ -1150,9 +1196,16 @@ impl TrustedRuntimeHost { .iter() .map(|(submission_id, _, outcome)| (*submission_id, outcome.clone())) .collect(); - self.echo_operation_action_admission_obstructions.clear(); self.runtime = restored_runtime; self.provenance = restored_provenance; + self.pending_echo_operation_actions = pending_echo_operation_action_ids_v1( + &self.runtime, + &self.echo_operation_action_outcomes, + ); + self.admitted_echo_operation_actions.clear(); + self.echo_operation_action_admission_obstructions.clear(); + #[cfg(any(test, feature = "host_test"))] + self.echo_operation_action_admission_attempts.clear(); self.runtime_wal = Some(runtime_wal); Ok(()) } @@ -1661,29 +1714,65 @@ impl TrustedRuntimeHost { ) } + fn track_pending_echo_operation_action_v1( + &mut self, + submission_id: Hash, + is_echo_operation_action: bool, + ) { + if is_echo_operation_action + && !self + .echo_operation_action_outcomes + .contains_key(&submission_id) + { + self.pending_echo_operation_actions.insert(submission_id); + } + } + + fn requeue_obstructed_echo_operation_actions_v1(&mut self) { + self.pending_echo_operation_actions.extend( + self.echo_operation_action_admission_obstructions + .keys() + .copied(), + ); + self.echo_operation_action_admission_obstructions.clear(); + } + fn admit_pending_echo_operation_actions_v1( &mut self, ) -> Result, TrustedRuntimeHostError> { - let pending = self - .runtime - .pending_witnessed_submissions() - .filter_map(|submission| { - self.runtime - .witnessed_submission_envelope(&submission.submission_id) - .and_then(|envelope| { - echo_operation_action_invocation_bytes_v1(envelope) - .map(|bytes| (submission.clone(), envelope.clone(), bytes.to_vec())) - }) - }) - .collect::>(); - if pending.is_empty() { - return Ok(BTreeMap::new()); + if self.pending_echo_operation_actions.is_empty() { + return Ok(self.admitted_echo_operation_actions.clone()); } let Some(policy) = self.echo_operation_action_admission_policy else { - return Ok(BTreeMap::new()); + return Ok(self.admitted_echo_operation_actions.clone()); }; - let mut admitted_actions = BTreeMap::new(); - for (submission, envelope, invocation_bytes) in pending { + let available = crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1 + .saturating_sub(self.admitted_echo_operation_actions.len()); + let pending = self + .pending_echo_operation_actions + .iter() + .filter(|submission_id| { + !self + .admitted_echo_operation_actions + .contains_key(*submission_id) + }) + .take(available) + .copied() + .collect::>(); + for submission_id in pending { + let submission = self + .runtime + .witnessed_submission(&submission_id) + .cloned() + .ok_or(RuntimeError::UnknownIntentSubmission(submission_id))?; + let envelope = self + .runtime + .witnessed_submission_envelope(&submission_id) + .cloned() + .ok_or(RuntimeError::UnknownIntentSubmission(submission_id))?; + let invocation_bytes = echo_operation_action_invocation_bytes_v1(&envelope) + .ok_or(RuntimeError::UnknownIntentSubmission(submission_id))? + .to_vec(); if let Some(runtime_wal) = self.runtime_wal.as_ref() { let durably_accepted = runtime_wal .has_submission_acceptance(submission.submission_id, submission.ingress_id)?; @@ -1691,17 +1780,18 @@ impl TrustedRuntimeHost { continue; } } - if self - .echo_operation_action_admission_obstructions - .contains_key(&submission.submission_id) - { - continue; - } + #[cfg(any(test, feature = "host_test"))] + self.echo_operation_action_admission_attempts + .entry(submission_id) + .and_modify(|attempts| *attempts += 1) + .or_insert(1); let package_id = match decode_invocation_route_v1(&invocation_bytes) { Ok((package_id, _)) => package_id, Err(error) => { self.echo_operation_action_admission_obstructions .insert(submission.submission_id, error.kind()); + self.pending_echo_operation_actions + .remove(&submission.submission_id); continue; } }; @@ -1715,6 +1805,8 @@ impl TrustedRuntimeHost { Err(error) => { self.echo_operation_action_admission_obstructions .insert(submission.submission_id, error.kind()); + self.pending_echo_operation_actions + .remove(&submission.submission_id); continue; } }; @@ -1727,9 +1819,10 @@ impl TrustedRuntimeHost { )?; self.echo_operation_action_admission_obstructions .remove(&submission.submission_id); - admitted_actions.insert(submission.submission_id, admitted); + self.admitted_echo_operation_actions + .insert(submission.submission_id, admitted); } - Ok(admitted_actions) + Ok(self.admitted_echo_operation_actions.clone()) } /// Runs one scheduler-owned pass. @@ -1867,6 +1960,10 @@ impl TrustedRuntimeHost { } } for (submission_id, outcome) in action_outcomes { + self.pending_echo_operation_actions.remove(&submission_id); + self.admitted_echo_operation_actions.remove(&submission_id); + self.echo_operation_action_admission_obstructions + .remove(&submission_id); self.echo_operation_action_outcomes .insert(submission_id, outcome); } @@ -5186,7 +5283,16 @@ impl TrustedRuntimeApp<'_> { &mut self, envelope: IngressEnvelope, ) -> Result { - self.host.runtime.submit_app_intent(envelope) + let is_echo_operation_action = + echo_operation_action_invocation_bytes_v1(&envelope).is_some(); + let handle = self.host.runtime.submit_app_intent(envelope)?; + if self.host.runtime_wal.is_none() { + self.host.track_pending_echo_operation_action_v1( + handle.submission_id, + is_echo_operation_action, + ); + } + Ok(handle) } /// Submits canonical intent material and returns only after the configured @@ -5218,6 +5324,8 @@ impl TrustedRuntimeApp<'_> { return Err(TrustedRuntimeHostError::RuntimeWalUnavailable); } + let is_echo_operation_action = + echo_operation_action_invocation_bytes_v1(&envelope).is_some(); let before_runtime = self.host.runtime.clone(); let handle = match admission { AppIntentAdmission::Ordinary => { @@ -5235,7 +5343,13 @@ impl TrustedRuntimeApp<'_> { if handle.duplicate { match runtime_wal.has_submission_acceptance(handle.submission_id, envelope.ingress_id()) { - Ok(true) => return Ok(handle), + Ok(true) => { + self.host.track_pending_echo_operation_action_v1( + handle.submission_id, + is_echo_operation_action, + ); + return Ok(handle); + } Ok(false) => {} Err(error) => { self.host.runtime = before_runtime; @@ -5248,11 +5362,17 @@ impl TrustedRuntimeApp<'_> { handle.submission_id, envelope.ingress_id(), ) { + self.host.track_pending_echo_operation_action_v1( + handle.submission_id, + is_echo_operation_action, + ); return Ok(handle); } self.host.runtime = before_runtime; return Err(error.into()); } + self.host + .track_pending_echo_operation_action_v1(handle.submission_id, is_echo_operation_action); Ok(handle) } diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 74f96ce1..132dc2e7 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2743,6 +2743,15 @@ fn budget_deferred_executable_action_reaches_typed_basis_obstruction() { host.echo_operation_action_outcome_v1(&committed_submission_id), Some(EchoOperationActionOutcomeV1::Committed(_)) )); + assert_eq!( + host.echo_operation_action_admission_attempts_for_test(&committed_submission_id), + 1 + ); + assert_eq!( + host.echo_operation_action_admission_attempts_for_test(&deferred_submission_id), + 1, + "the host reuses admission evidence while the inbox defers an Action" + ); } #[test] @@ -3065,6 +3074,9 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { assert!(submission_ids.iter().all(|submission_id| host .echo_operation_action_outcome_v1(submission_id) .is_some())); + assert!(submission_ids.iter().all(|submission_id| { + host.echo_operation_action_admission_attempts_for_test(submission_id) == 1 + })); } #[test] @@ -3405,6 +3417,19 @@ fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { .echo_operation_action_outcome_v1(&unavailable_submission_id) .is_none()); assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); + assert_eq!( + host.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), + 1 + ); + assert!(host + .tick_once() + .expect("quarantined admission work does not poison an idle pass") + .is_empty()); + assert_eq!( + host.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), + 1, + "a quarantined Action is not decoded and re-admitted every Tick" + ); } let (mut recovered, recovered_head, recovered_node) = fixture_host(); @@ -3424,6 +3449,18 @@ fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { recovered.echo_operation_action_admission_obstruction_v1(&unavailable_submission_id), Some(EchoOperationInvocationAdmissionErrorKindV1::OperationUnavailable) ); + assert_eq!( + recovered.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), + 1 + ); + assert!(recovered + .tick_once() + .expect("recovered quarantine remains idle without re-admission") + .is_empty()); + assert_eq!( + recovered.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), + 1 + ); assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 1); assert_eq!(recovered_head, head_key); assert_eq!(recovered_node, node); From 47a98a13d993643777c9b8111007a972a3e014e7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:26:51 -0700 Subject: [PATCH 27/64] Fix: preserve legacy operation recovery index roots --- crates/warp-core/src/trusted_runtime_host.rs | 84 ++++++++++++++++++-- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 4968303e..8fdc3b70 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -5808,6 +5808,60 @@ mod tests { .expect("the recovery parent-state fixture is lawful") } + #[test] + fn executable_operation_index_preserves_legacy_root_without_action_outcomes() { + let operation_coordinate = "echo.test.LegacyRecoveryIndex.v1"; + let authority_profile_identity = [0x17; 32]; + let budget = crate::EchoOperationBudgetV1::new(7, 1_024, 1_024); + let package = crate::ExecutableOperationPackageV1::new( + operation_coordinate, + crate::EchoOperationSemanticClosureV1::new( + [0x10; 32], + [0x11; 32], + [0x12; 32], + [0x13; 32], + "echo.test.legacy-index-schema/v1", + [0x14; 32], + "echo.test.legacy-index-lawpack/v1", + [0x15; 32], + ), + crate::echo_operation_target_profile_identity_v1(), + authority_profile_identity, + budget, + crate::EchoOperationProgramV1::anchored_node_attachment_compare_and_set( + crate::make_type_id("legacy-index-node"), + crate::make_type_id("legacy-index-attachment"), + 128, + ), + ); + let package_bytes = package + .to_canonical_bytes() + .expect("the legacy-index package is canonical"); + let package_id = crate::echo_operation_package_id_v1(&package_bytes); + let admitted = admit_package_v1( + &crate::EchoOperationAdmissionPolicyV1::exact( + package_id, + operation_coordinate, + authority_profile_identity, + budget, + ), + package_bytes, + ) + .expect("the legacy-index package is admitted"); + let installed = + installed_from_admitted(admitted).expect("the legacy-index package installs"); + + assert_eq!( + recovered_echo_operation_index_root([0x20; 32], &[installed], &[], &[]) + .expect("the legacy index root is computable"), + [ + 0xff, 0x77, 0x8e, 0x79, 0x1c, 0x4b, 0x7f, 0x99, 0xb7, 0xa5, 0x4c, 0x4b, 0x8d, 0xdb, + 0x36, 0x91, 0x62, 0x17, 0x8a, 0x22, 0xe2, 0xbe, 0xde, 0xc6, 0x54, 0x4c, 0x8d, 0x25, + 0xa1, 0xa0, 0x69, 0x04, + ] + ); + } + #[test] fn creation_wal_scope_accepts_descendants_and_rejects_mutated_shapes() { let node = crate::NodeKey { @@ -7186,7 +7240,29 @@ fn recovered_echo_operation_index_root( receipts: &[EchoOperationReceiptV1], action_outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], ) -> Result { - if installations.is_empty() && receipts.is_empty() && action_outcomes.is_empty() { + let legacy_root = + recovered_echo_operation_legacy_index_root(base_root, installations, receipts)?; + if action_outcomes.is_empty() { + return Ok(legacy_root); + } + let mut hasher = blake3::Hasher::new(); + hasher.update(b"echo:trusted-runtime-wal:executable-operation-index:v2\0"); + hasher.update(&legacy_root); + hasher.update(&(action_outcomes.len() as u64).to_le_bytes()); + for (submission_id, ingress_id, outcome) in action_outcomes { + let bytes = retain_action_outcome_v1(*submission_id, *ingress_id, outcome)?; + hasher.update(&(bytes.len() as u64).to_le_bytes()); + hasher.update(&bytes); + } + Ok(hasher.finalize().into()) +} + +fn recovered_echo_operation_legacy_index_root( + base_root: Hash, + installations: &[InstalledEchoOperationV1], + receipts: &[EchoOperationReceiptV1], +) -> Result { + if installations.is_empty() && receipts.is_empty() { return Ok(base_root); } let mut hasher = blake3::Hasher::new(); @@ -7204,12 +7280,6 @@ fn recovered_echo_operation_index_root( hasher.update(&(bytes.len() as u64).to_le_bytes()); hasher.update(&bytes); } - hasher.update(&(action_outcomes.len() as u64).to_le_bytes()); - for (submission_id, ingress_id, outcome) in action_outcomes { - let bytes = retain_action_outcome_v1(*submission_id, *ingress_id, outcome)?; - hasher.update(&(bytes.len() as u64).to_le_bytes()); - hasher.update(&bytes); - } Ok(hasher.finalize().into()) } From 3ca7e21cd5cba5b67a12b7e4288608af01bf8e28 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:33:21 -0700 Subject: [PATCH 28/64] Fix: separate production and instrumented Action tests --- .github/workflows/ci.yml | 8 +- .github/workflows/macos-local.yml | 8 +- crates/warp-core/Cargo.toml | 2 +- crates/warp-core/src/trusted_runtime_host.rs | 24 -- .../executable_operation_pipeline_tests.rs | 244 +++++++++--------- scripts/verify-local.sh | 17 +- 6 files changed, 145 insertions(+), 158 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4834b00e..57dbe01f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,7 +146,9 @@ jobs: cargo clippy -p warp-core --features host_test --test causal_fact_publication_tests -- -D warnings -D missing_docs cargo clippy -p warp-core --features host_test --test optic_invocation_admission_tests -- -D warnings -D missing_docs - name: cargo clippy (scheduler-owned executable Actions) - run: cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs + run: | + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests -- -D warnings -D missing_docs + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs clippy-det-fixed: name: Clippy (det_fixed) @@ -304,7 +306,9 @@ jobs: - name: cargo test (warp-core provider proposal admission) run: cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime --test provider_contract_admission_tests - name: cargo test (scheduler-owned executable Actions) - run: cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests + run: | + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests - name: cargo test --doc (warp-core) run: cargo test -p warp-core --doc - name: PRNG golden regression (warp-math) diff --git a/.github/workflows/macos-local.yml b/.github/workflows/macos-local.yml index 62b7f3ec..e1d52b78 100644 --- a/.github/workflows/macos-local.yml +++ b/.github/workflows/macos-local.yml @@ -52,7 +52,9 @@ jobs: cargo clippy -p warp-core --features host_test --test causal_fact_publication_tests -- -D warnings -D missing_docs cargo clippy -p warp-core --features host_test --test optic_invocation_admission_tests -- -D warnings -D missing_docs - name: cargo clippy (scheduler-owned executable Actions) - run: cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs + run: | + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests -- -D warnings -D missing_docs + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs - name: cargo test (workspace) run: cargo test --workspace - name: cargo test (warp-core host_test admission fixtures) @@ -62,4 +64,6 @@ jobs: - name: cargo test (warp-core installed contract intent pipeline) run: cargo test -p warp-core --features native_rule_bootstrap,host_test --test installed_contract_intent_pipeline_tests - name: cargo test (scheduler-owned executable Actions) - run: cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests + run: | + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests diff --git a/crates/warp-core/Cargo.toml b/crates/warp-core/Cargo.toml index 20197b3d..7504ab5b 100644 --- a/crates/warp-core/Cargo.toml +++ b/crates/warp-core/Cargo.toml @@ -95,7 +95,7 @@ required-features = ["native_rule_bootstrap", "trusted_runtime"] [[test]] name = "executable_operation_pipeline_tests" -required-features = ["native_rule_bootstrap", "trusted_runtime", "host_test"] +required-features = ["native_rule_bootstrap", "trusted_runtime"] [build-dependencies] blake3 = "1.0" diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 8fdc3b70..26d0a68a 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -717,8 +717,6 @@ pub struct TrustedRuntimeHost { admitted_echo_operation_actions: BTreeMap, echo_operation_action_admission_obstructions: BTreeMap, - #[cfg(any(test, feature = "host_test"))] - echo_operation_action_admission_attempts: BTreeMap, } fn pending_echo_operation_action_ids_v1( @@ -763,8 +761,6 @@ impl TrustedRuntimeHost { pending_echo_operation_actions, admitted_echo_operation_actions: BTreeMap::new(), echo_operation_action_admission_obstructions: BTreeMap::new(), - #[cfg(any(test, feature = "host_test"))] - echo_operation_action_admission_attempts: BTreeMap::new(), }) } @@ -789,8 +785,6 @@ impl TrustedRuntimeHost { pending_echo_operation_actions, admitted_echo_operation_actions: BTreeMap::new(), echo_operation_action_admission_obstructions: BTreeMap::new(), - #[cfg(any(test, feature = "host_test"))] - echo_operation_action_admission_attempts: BTreeMap::new(), } } @@ -850,17 +844,6 @@ impl TrustedRuntimeHost { .copied() } - /// Returns how often runtime-owned admission evaluated one Action in - /// scheduler tests. - #[cfg(any(test, feature = "host_test"))] - #[must_use] - pub fn echo_operation_action_admission_attempts_for_test(&self, submission_id: &Hash) -> u64 { - self.echo_operation_action_admission_attempts - .get(submission_id) - .copied() - .unwrap_or(0) - } - /// Admits exact executable-operation package bytes under independent policy. /// /// A successful result is an opaque admission token. It does not install @@ -1204,8 +1187,6 @@ impl TrustedRuntimeHost { ); self.admitted_echo_operation_actions.clear(); self.echo_operation_action_admission_obstructions.clear(); - #[cfg(any(test, feature = "host_test"))] - self.echo_operation_action_admission_attempts.clear(); self.runtime_wal = Some(runtime_wal); Ok(()) } @@ -1780,11 +1761,6 @@ impl TrustedRuntimeHost { continue; } } - #[cfg(any(test, feature = "host_test"))] - self.echo_operation_action_admission_attempts - .entry(submission_id) - .and_modify(|attempts| *attempts += 1) - .or_insert(1); let package_id = match decode_invocation_route_v1(&invocation_bytes) { Ok((package_id, _)) => package_id, Err(error) => { diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 132dc2e7..91980130 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -12,25 +12,28 @@ use std::{ use bytes::Bytes; use echo_edict_canonical::{decode_canonical_cbor_v1, encode_canonical_cbor_v1, CanonicalValueV1}; use warp_core::causal_wal::{ - AffectedFrontier, AffectedFrontierKind, FilesystemWalFaultPlan, FilesystemWalFaultTarget, Lsn, - PayloadCodecId, PayloadSchemaId, WalAppendAuthority, WalBuildError, WalDurabilityMode, - WalRecordKind, WalSegmentId, WalTickDecision, WalTransactionBuilder, WalTransactionId, - WalTransactionKind, WalValidationError, WriterEpochId, + AffectedFrontier, AffectedFrontierKind, Lsn, PayloadCodecId, PayloadSchemaId, + WalAppendAuthority, WalBuildError, WalDurabilityMode, WalRecordKind, WalSegmentId, + WalTransactionBuilder, WalTransactionId, WalTransactionKind, WalValidationError, WriterEpochId, }; +#[cfg(feature = "host_test")] +use warp_core::causal_wal::{FilesystemWalFaultPlan, FilesystemWalFaultTarget, WalTickDecision}; use warp_core::{ make_head_id, make_node_id, make_type_id, make_warp_id, AtomPayload, AttachmentValue, EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorKindV1, EchoOperationAdmissionPolicyV1, EchoOperationAnchoredNodeOccupancyV1, EchoOperationApplicationBasisV1, EchoOperationArtifactErrorKindV1, EchoOperationBudgetV1, - EchoOperationCommitErrorV1, EchoOperationInvocationAdmissionErrorKindV1, - EchoOperationInvocationAdmissionPolicyV1, EchoOperationInvocationV1, - EchoOperationObstructionKindV1, EchoOperationPreparationV1, EchoOperationProgramV1, - EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, EngineBuilder, - ExecutableOperationPackageV1, GraphStore, InboxPolicy, IngressTarget, InstalledEchoOperationV1, - NodeKey, NodeRecord, PlaybackMode, RuntimeError, RuntimeWalActivationGap, SchedulerKind, - TrustedRuntimeHost, TrustedRuntimeHostError, TrustedRuntimeWalConfig, TrustedRuntimeWalError, - WorldlineId, WorldlineRuntime, WorldlineState, WriterHead, WriterHeadKey, + EchoOperationInvocationAdmissionErrorKindV1, EchoOperationInvocationAdmissionPolicyV1, + EchoOperationInvocationV1, EchoOperationObstructionKindV1, EchoOperationPreparationV1, + EchoOperationProgramV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, + EngineBuilder, ExecutableOperationPackageV1, GraphStore, InboxPolicy, IngressTarget, + InstalledEchoOperationV1, NodeKey, NodeRecord, PlaybackMode, RuntimeWalActivationGap, + SchedulerKind, TrustedRuntimeHost, TrustedRuntimeHostError, TrustedRuntimeWalConfig, + TrustedRuntimeWalError, WorldlineId, WorldlineRuntime, WorldlineState, WriterHead, + WriterHeadKey, }; +#[cfg(feature = "host_test")] +use warp_core::{EchoOperationCommitErrorV1, RuntimeError}; const OPERATION_COORDINATE: &str = "echo.fixture.SetAnchoredAtom.v1"; const CREATE_OPERATION_COORDINATE: &str = "echo.fixture.CreateAnchoredAtomIfAbsent.v1"; @@ -2743,15 +2746,6 @@ fn budget_deferred_executable_action_reaches_typed_basis_obstruction() { host.echo_operation_action_outcome_v1(&committed_submission_id), Some(EchoOperationActionOutcomeV1::Committed(_)) )); - assert_eq!( - host.echo_operation_action_admission_attempts_for_test(&committed_submission_id), - 1 - ); - assert_eq!( - host.echo_operation_action_admission_attempts_for_test(&deferred_submission_id), - 1, - "the host reuses admission evidence while the inbox defers an Action" - ); } #[test] @@ -2906,67 +2900,74 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { "one composite Tick retains one decision record, per-Action outcomes, and one state delta" ); - let mut adversarial = runtime_wal - .recover_read_only() - .expect("the honest composite Tick recovers"); - let mut missing_installation = adversarial.clone(); - missing_installation.installed_echo_operations.clear(); - assert!(matches!( - missing_installation.validate_echo_operation_action_outcomes_for_test(), - Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) - )); - let mut contradictory_decision = adversarial.clone(); - contradictory_decision.replace_echo_operation_action_decision_for_test( - first_submission_id, - WalTickDecision::Obstructed, - ); - assert!(matches!( - contradictory_decision.validate_echo_operation_action_outcomes_for_test(), - Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) - )); - let mut forged_admission_ticket = adversarial.clone(); - let forged_ticket = digest("forged-Action-admission-ticket"); - forged_admission_ticket.receipt_correlations[0].ticket_digest = forged_ticket; - forged_admission_ticket.receipt_correlations[0] - .causal_receipt_ref - .ticket_digest = forged_ticket; - assert!(matches!( - forged_admission_ticket.validate_echo_operation_action_outcomes_for_test(), - Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) - )); - let mut retroactive_installation = adversarial.clone(); - retroactive_installation - .clear_echo_operation_action_installations_before_tick_for_test(first_submission_id); - assert!(matches!( - retroactive_installation.validate_echo_operation_action_outcomes_for_test(), - Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) - )); - let mut forged_composition = adversarial.clone(); - let Some((_, _, EchoOperationActionOutcomeV1::Committed(receipt))) = forged_composition - .echo_operation_action_outcomes - .iter_mut() - .find(|(_, _, outcome)| matches!(outcome, EchoOperationActionOutcomeV1::Committed(_))) - else { - panic!("the recovered composite Tick retains one committed Action receipt"); - }; - receipt.replace_composition_digest_for_test(digest("forged-composition")); - assert!(matches!( - forged_composition.validate_echo_operation_action_outcomes_for_test(), - Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) - )); - assert!( - adversarial + #[cfg(feature = "host_test")] + { + let mut adversarial = runtime_wal + .recover_read_only() + .expect("the honest composite Tick recovers"); + let mut missing_installation = adversarial.clone(); + missing_installation.installed_echo_operations.clear(); + assert!(matches!( + missing_installation.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut contradictory_decision = adversarial.clone(); + contradictory_decision.replace_echo_operation_action_decision_for_test( + first_submission_id, + WalTickDecision::Obstructed, + ); + assert!(matches!( + contradictory_decision.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut forged_admission_ticket = adversarial.clone(); + let forged_ticket = digest("forged-Action-admission-ticket"); + forged_admission_ticket.receipt_correlations[0].ticket_digest = forged_ticket; + forged_admission_ticket.receipt_correlations[0] + .causal_receipt_ref + .ticket_digest = forged_ticket; + assert!(matches!( + forged_admission_ticket.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut retroactive_installation = adversarial.clone(); + retroactive_installation + .clear_echo_operation_action_installations_before_tick_for_test( + first_submission_id, + ); + assert!(matches!( + retroactive_installation.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut forged_composition = adversarial.clone(); + let Some((_, _, EchoOperationActionOutcomeV1::Committed(receipt))) = forged_composition .echo_operation_action_outcomes - .windows(2) - .all(|pair| pair[0].1 < pair[1].1), - "Action outcome records are retained in canonical ingress order" - ); - let (first, second) = adversarial.echo_operation_action_outcomes.split_at_mut(1); - std::mem::swap(&mut first[0].2, &mut second[0].2); - assert!(matches!( - adversarial.validate_echo_operation_action_outcomes_for_test(), - Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) - )); + .iter_mut() + .find(|(_, _, outcome)| { + matches!(outcome, EchoOperationActionOutcomeV1::Committed(_)) + }) + else { + panic!("the recovered composite Tick retains one committed Action receipt"); + }; + receipt.replace_composition_digest_for_test(digest("forged-composition")); + assert!(matches!( + forged_composition.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + assert!( + adversarial + .echo_operation_action_outcomes + .windows(2) + .all(|pair| pair[0].1 < pair[1].1), + "Action outcome records are retained in canonical ingress order" + ); + let (first, second) = adversarial.echo_operation_action_outcomes.split_at_mut(1); + std::mem::swap(&mut first[0].2, &mut second[0].2); + assert!(matches!( + adversarial.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + } } let (mut recovered, head_key, first_node, second_node) = fixture_host_with_bare_node( @@ -3074,9 +3075,6 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { assert!(submission_ids.iter().all(|submission_id| host .echo_operation_action_outcome_v1(submission_id) .is_some())); - assert!(submission_ids.iter().all(|submission_id| { - host.echo_operation_action_admission_attempts_for_test(submission_id) == 1 - })); } #[test] @@ -3417,19 +3415,10 @@ fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { .echo_operation_action_outcome_v1(&unavailable_submission_id) .is_none()); assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); - assert_eq!( - host.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), - 1 - ); assert!(host .tick_once() .expect("quarantined admission work does not poison an idle pass") .is_empty()); - assert_eq!( - host.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), - 1, - "a quarantined Action is not decoded and re-admitted every Tick" - ); } let (mut recovered, recovered_head, recovered_node) = fixture_host(); @@ -3449,18 +3438,10 @@ fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { recovered.echo_operation_action_admission_obstruction_v1(&unavailable_submission_id), Some(EchoOperationInvocationAdmissionErrorKindV1::OperationUnavailable) ); - assert_eq!( - recovered.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), - 1 - ); assert!(recovered .tick_once() .expect("recovered quarantine remains idle without re-admission") .is_empty()); - assert_eq!( - recovered.echo_operation_action_admission_attempts_for_test(&unavailable_submission_id), - 1 - ); assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 1); assert_eq!(recovered_head, head_key); assert_eq!(recovered_node, node); @@ -3681,33 +3662,38 @@ fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { }) .expect("exactly one Action is rejected by the earlier applied footprint"); - let mut adversarial = host - .runtime_wal() - .expect("the conflict WAL remains enabled") - .recover_read_only() - .expect("the honest conflict batch recovers"); - adversarial.replace_echo_operation_action_conflict_preparation_for_test( - conflict_submission_id, - digest("forged-conflict-preparation"), - ); - assert!(matches!( - adversarial.validate_echo_operation_action_outcomes_for_test(), - Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) - )); - let mut false_conflict = host - .runtime_wal() - .expect("the conflict WAL remains enabled") - .recover_read_only() - .expect("the honest conflict batch recovers again"); - false_conflict.replace_echo_operation_action_conflict_blockers_for_test( - conflict_submission_id, - Vec::new(), - ); - assert!(matches!( - false_conflict - .validate_echo_operation_parent_states_for_test(host.runtime(), host.provenance()), - Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) - )); + #[cfg(feature = "host_test")] + { + let mut adversarial = host + .runtime_wal() + .expect("the conflict WAL remains enabled") + .recover_read_only() + .expect("the honest conflict batch recovers"); + adversarial.replace_echo_operation_action_conflict_preparation_for_test( + conflict_submission_id, + digest("forged-conflict-preparation"), + ); + assert!(matches!( + adversarial.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut false_conflict = host + .runtime_wal() + .expect("the conflict WAL remains enabled") + .recover_read_only() + .expect("the honest conflict batch recovers again"); + false_conflict.replace_echo_operation_action_conflict_blockers_for_test( + conflict_submission_id, + Vec::new(), + ); + assert!(matches!( + false_conflict.validate_echo_operation_parent_states_for_test( + host.runtime(), + host.provenance() + ), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); + } } let (mut recovered, _, _) = fixture_host(); @@ -3720,6 +3706,7 @@ fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { )); } +#[cfg(feature = "host_test")] #[test] fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { let wal_dir = TempWalDir::new(); @@ -3810,6 +3797,7 @@ fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { )); } +#[cfg(feature = "host_test")] #[test] fn scheduler_tick_construction_failure_publishes_no_action_state_or_receipt() { let (mut host, head_key, node) = fixture_host(); diff --git a/scripts/verify-local.sh b/scripts/verify-local.sh index f1e080eb..25a15c0e 100755 --- a/scripts/verify-local.sh +++ b/scripts/verify-local.sh @@ -1161,7 +1161,7 @@ pre_push_feature_string_for_test_target() { printf '%s\n' "native_rule_bootstrap,trusted_runtime" ;; warp-core:executable_operation_pipeline_tests) - printf '%s\n' "native_rule_bootstrap,trusted_runtime,host_test" + printf '%s\n' "native_rule_bootstrap,trusted_runtime" ;; warp-core:installed_contract_registry_tests) printf '%s\n' "native_rule_bootstrap" @@ -1213,6 +1213,11 @@ collect_pre_push_rust_slices() { target="$(basename "$file" .rs)" features="$(pre_push_feature_string_for_test_target "$crate" "$target")" append_pre_push_rust_slice "${crate}|test|${target}|${features}|" slices + if [[ "${crate}:${target}" == "warp-core:executable_operation_pipeline_tests" ]]; then + append_pre_push_rust_slice \ + "${crate}|test|${target}|native_rule_bootstrap,trusted_runtime,host_test|" \ + slices + fi ;; crates/*/src/lib.rs) crate="$(printf '%s\n' "$file" | sed -n 's#^crates/\([^/]*\)/.*#\1#p')" @@ -1435,6 +1440,11 @@ run_warp_core_test_target() { local -a feature_args=() mapfile -t feature_args < <(warp_core_feature_args_for_test "$test_target") lane_cargo "$lane" test -p warp-core "${feature_args[@]}" --test "$test_target" + if [[ "$test_target" == "executable_operation_pipeline_tests" ]]; then + lane_cargo "$lane" test -p warp-core \ + --features native_rule_bootstrap,trusted_runtime,host_test \ + --test "$test_target" + fi } run_warp_core_clippy_test_target() { @@ -1443,6 +1453,11 @@ run_warp_core_clippy_test_target() { local -a feature_args=() mapfile -t feature_args < <(warp_core_feature_args_for_test "$test_target") lane_cargo "$lane" clippy -p warp-core "${feature_args[@]}" --test "$test_target" -- -D warnings -D missing_docs + if [[ "$test_target" == "executable_operation_pipeline_tests" ]]; then + lane_cargo "$lane" clippy -p warp-core \ + --features native_rule_bootstrap,trusted_runtime,host_test \ + --test "$test_target" -- -D warnings -D missing_docs + fi } prepare_warp_wasm_scope() { From 0f56b622c46d9f621f6cb983f863bc45d87fa247 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:35:25 -0700 Subject: [PATCH 29/64] Docs: state direct operation seam transition exactly --- ...3-admitted-executable-operation-packages.md | 6 ++++-- ...duler-owned-executable-operation-actions.md | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/adr/0023-admitted-executable-operation-packages.md b/docs/adr/0023-admitted-executable-operation-packages.md index 07e40dd1..77986abd 100644 --- a/docs/adr/0023-admitted-executable-operation-packages.md +++ b/docs/adr/0023-admitted-executable-operation-packages.md @@ -12,8 +12,10 @@ > invocation, private-evaluation, and recovery semantics. ADR 0025 supersedes > direct prepare/commit as the application lifecycle: canonical executable > Actions now enter ordinary durable ingress, and only the scheduler invokes -> private evaluation while constructing an atomic Tick. The direct methods are -> transitional compatibility and test seams. +> private evaluation while constructing an atomic Tick. The direct methods +> remain documentation-hidden public `TrustedRuntimeHost` compatibility/test +> seams; they are absent from `TrustedRuntimeApp`, and their eventual removal is +> tracked by [issue #689](https://github.com/flyingrobots/echo/issues/689). ## Context diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md index bb1359e5..6ecc79b1 100644 --- a/docs/adr/0025-scheduler-owned-executable-operation-actions.md +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -71,10 +71,13 @@ worldline: - neither result mutates the parent state. The direct `prepare_echo_operation_v1` and -`commit_prepared_echo_operation_v1` methods remain temporarily available only -as explicitly transitional compatibility/test seams. They are not the -application architecture and will be removed after existing convergence -witnesses migrate. +`commit_prepared_echo_operation_v1` methods remain publicly reachable on +`TrustedRuntimeHost` as explicitly transitional, documentation-hidden +compatibility/test seams. `TrustedRuntimeApp` does not expose them, but that +application-facing type boundary is not a claim that Rust visibility prevents +every downstream host owner from calling the methods. They are not the +application architecture. Their migration and removal are tracked by +[issue #689](https://github.com/flyingrobots/echo/issues/689). ### Legacy and executable ingress never share an evaluator batch @@ -151,7 +154,10 @@ and receipt correlation. ## Consequences -- Application code cannot invoke executable evaluation or direct commit. +- The application-facing `TrustedRuntimeApp` cannot invoke executable + evaluation or direct commit. Downstream code that owns a + `TrustedRuntimeHost` can still reach the documentation-hidden transitional + methods until issue #689 removes that exception. - Two independent executable-operation Actions can contribute to one scheduler-owned Tick. - A typed obstruction is durable evidence and cannot hide a state mutation. @@ -177,6 +183,8 @@ The implementation is accepted only with executable witnesses that prove: 9. Fresh-host recovery reconstructs Action outcomes, Tick, state, and receipts. 10. Direct prepare/commit is marked transitional and absent from the application-facing surface. +11. Removal of the remaining public host-owner compatibility seams is tracked + separately rather than claimed complete by this decision. ## Non-Goals From b668ba4d0e1566105156d10b53e73bf480e2195c Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:36:33 -0700 Subject: [PATCH 30/64] Docs: record scheduler Action recovery guarantees --- CHANGELOG.md | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 633604b2..6c51c78d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,19 +10,27 @@ - Executable-operation application writes now enter Echo as canonical, WAL-acknowledged Actions and are evaluated only by the scheduler while constructing a Tick (ADR 0025). Accepted pre-Tick Actions recover as pending - work. Two independent Actions can share one exact parent coordinate and - contribute to one composite Tick, while retaining candidate-specific - application-basis propositions and per-Action typed outcomes. Footprint - conflicts name earlier applied members; evaluator obstructions contribute no - operations. One scheduler WAL transaction retains exactly one batched Tick - decision record, then each Action's receipt correlation and typed outcome in - canonical order, followed by exactly one replayable state delta. The decided - Tick is durable before state, frontier, receipt, or outcome publication, and - fresh-host recovery validates every - outcome against its exact envelope, invocation, installed operation, Tick - entry, composite consequence, and state root. Direct operation - prepare/commit remains a hidden transitional compatibility/test seam rather - than an application execution lifecycle. + work. Runtime-owned admission uses a bounded pending index and cache; + unavailable packages are quarantined without poisoning unrelated work. + Scheduler selection admits at most 64 executable Actions per Tick, leaving + excess work pending, and meters footprint comparisons, blocker evidence, and + aggregate operations during composition. Two independent Actions can share + one exact parent coordinate and contribute to one composite Tick while + retaining candidate-specific application-basis propositions and per-Action + typed outcomes. Footprint conflicts name earlier applied members; evaluator + and composition-budget obstructions contribute no operations. + One scheduler WAL transaction retains exactly one batched Tick decision + record, then each Action's receipt correlation and typed outcome in canonical + order, followed by exactly one replayable state delta. The decided Tick is + durable before state, frontier, receipt, or outcome publication. Fresh-host + recovery validates every outcome against its exact envelope, admission, + invocation, installed operation, causal coordinate, evaluation basis, + reconstructed preparation and actual footprint, Tick entry, composite + consequence, and state root. Legacy operation recovery-index roots remain + byte-compatible when no Action outcome exists. Direct operation + prepare/commit remains a documentation-hidden public + `TrustedRuntimeHost` compatibility/test seam, absent from + `TrustedRuntimeApp`; removal is tracked by issue #689. - `TrustedRuntimeHost` now has the first hook-free executable-operation runtime slice. A runtime owner can admit exact canonical `ExecutableOperationPackageV1` bytes under a separate package policy, install From 8006c2f843659f3012f495e4972fbb5f84e9743c Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:53:01 -0700 Subject: [PATCH 31/64] Fix: verify composite Action patch membership --- CHANGELOG.md | 6 +- crates/warp-core/src/echo_operation.rs | 73 ++++++++++++++----- crates/warp-core/src/trusted_runtime_host.rs | 69 +++++++++++++++++- .../executable_operation_pipeline_tests.rs | 22 ++++++ 4 files changed, 146 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c51c78d..678d9b76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,10 @@ recovery validates every outcome against its exact envelope, admission, invocation, installed operation, causal coordinate, evaluation basis, reconstructed preparation and actual footprint, Tick entry, composite - consequence, and state root. Legacy operation recovery-index roots remain - byte-compatible when no Action outcome exists. Direct operation + consequence, exact reconstructed aggregate patch membership, and state root. + A composite receipt cannot validate outside its complete Action-batch + context. Legacy operation recovery-index roots remain byte-compatible when + no Action outcome exists. Direct operation prepare/commit remains a documentation-hidden public `TrustedRuntimeHost` compatibility/test seam, absent from `TrustedRuntimeApp`; removal is tracked by issue #689. diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 49c04795..5dc99d07 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -3060,7 +3060,7 @@ pub(crate) fn recover_action_outcome_v1( .ok_or_else(|| invalid_structure("Action receipt bytes are truncated"))?; offset = end; EchoOperationActionOutcomeV1::Committed(Box::new( - EchoOperationReceiptV1::from_canonical_bytes(receipt_bytes)?, + EchoOperationReceiptV1::from_action_batch_canonical_bytes(receipt_bytes)?, )) } 2 => { @@ -4018,6 +4018,19 @@ impl EchoOperationReceiptV1 { } pub(crate) fn from_canonical_bytes(bytes: &[u8]) -> Result { + Self::decode_canonical_bytes(bytes, false) + } + + fn from_action_batch_canonical_bytes( + bytes: &[u8], + ) -> Result { + Self::decode_canonical_bytes(bytes, true) + } + + fn decode_canonical_bytes( + bytes: &[u8], + allow_composite_context: bool, + ) -> Result { let value = decode_canonical_cbor_v1(bytes).map_err(canonical_error)?; let mut fields = exact_text_map( value, @@ -4283,6 +4296,8 @@ impl EchoOperationReceiptV1 { || receipt.committed_patch_digest.is_none() || receipt.committed_result_id != Some(receipt.prepared_result_id) || receipt.composition_digest.is_none() + || (receipt.committed_patch_digest != Some(receipt.prepared_patch_digest) + && !allow_composite_context) || (receipt.committed_patch_digest == Some(receipt.prepared_patch_digest) && receipt.composition_digest != Some(expected_composition_digest)) || receipt.state_root_before != receipt.evaluation_basis.state_root @@ -4530,9 +4545,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( let mut blocked_by = Vec::with_capacity(candidates.len()); let mut decisions = Vec::with_capacity(candidates.len()); let mut accepted_footprints: Vec<(u32, Footprint)> = Vec::new(); - let mut in_slots = Vec::new(); - let mut out_slots = Vec::new(); - let mut ops = Vec::new(); + let mut accepted_operation_count = 0_usize; let mut ordered_rule_ids = Vec::with_capacity(candidates.len()); let mut footprint_comparisons = 0_usize; let mut blocker_evidence_count = 0_usize; @@ -4582,8 +4595,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( .checked_add(blockers.len()) .is_none_or(|total| total > ACTION_BATCH_BLOCKER_EVIDENCE_LIMIT_V1); let operation_budget_exceeded = blockers.is_empty() - && ops - .len() + && accepted_operation_count .checked_add(prepared.patch().ops().len()) .is_none_or(|total| total > ACTION_BATCH_OPERATION_LIMIT_V1); if comparison_budget_exceeded @@ -4617,9 +4629,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( blocked_by.push(Vec::new()); accepted_footprints .push((entry_index_u32, prepared.actual_footprint().clone())); - in_slots.extend_from_slice(prepared.patch().in_slots()); - out_slots.extend_from_slice(prepared.patch().out_slots()); - ops.extend_from_slice(prepared.patch().ops()); + accepted_operation_count += prepared.patch().ops().len(); decisions.push(( candidate.submission_id, SchedulerEchoOperationDecisionV1::Applied(prepared), @@ -4662,15 +4672,6 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( } } - let rule_pack_id = action_batch_rule_pack_id(&ordered_rule_ids); - let patch = WarpTickPatchV1::new( - policy_id, - rule_pack_id, - TickCommitStatus::Committed, - in_slots, - out_slots, - ops, - ); let tick_receipt = TickReceipt::new(tx, entries, blocked_by); let plan_digest = action_batch_plan_digest(tick_receipt.entries()); let applied = decisions @@ -4681,6 +4682,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( | SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(_) => None, }) .collect::>(); + let patch = action_batch_patch_from_preparations_v1(policy_id, &ordered_rule_ids, &applied); let rewrites_digest = action_batch_rewrites_digest(&applied); let composition_digest = action_batch_composition_digest(&applied, patch.digest()); @@ -4761,6 +4763,29 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( }) } +pub(crate) fn action_batch_patch_from_preparations_v1( + policy_id: u32, + ordered_rule_ids: &[Hash], + prepared: &[&PreparedEchoOperationV1], +) -> WarpTickPatchV1 { + let mut in_slots = Vec::new(); + let mut out_slots = Vec::new(); + let mut ops = Vec::new(); + for member in prepared { + in_slots.extend_from_slice(member.patch().in_slots()); + out_slots.extend_from_slice(member.patch().out_slots()); + ops.extend_from_slice(member.patch().ops()); + } + WarpTickPatchV1::new( + policy_id, + action_batch_rule_pack_id(ordered_rule_ids), + TickCommitStatus::Committed, + in_slots, + out_slots, + ops, + ) +} + fn action_batch_rule_pack_id(rule_ids: &[Hash]) -> Hash { let mut hasher = Hasher::new(); hasher.update(ACTION_BATCH_RULE_PACK_DOMAIN); @@ -6490,6 +6515,18 @@ mod tests { recover_committed_execution_receipt_v1(&forged_composition_bytes) .expect_err("recovery must independently derive singleton composition identity"); + let mut contextless_composite = receipt.clone(); + contextless_composite.committed_patch_digest = Some(digest(92)); + contextless_composite.composition_digest = Some(digest(93)); + contextless_composite.terminal_outcome_digest = + terminal_outcome_digest(&contextless_composite); + contextless_composite.receipt_digest = receipt_digest(&contextless_composite); + let contextless_composite_bytes = contextless_composite + .to_canonical_bytes() + .expect("contextless composite receipt encodes"); + recover_committed_execution_receipt_v1(&contextless_composite_bytes) + .expect_err("a composite receipt requires its complete scheduler batch context"); + let mut impossible_budget = receipt.clone(); impossible_budget.delegated_budget = EchoOperationBudgetV1::new(1, 1, 1); impossible_budget.receipt_digest = receipt_digest(&impossible_budget); diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 26d0a68a..c22aba82 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -43,9 +43,9 @@ use crate::{ contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ action_admission_evidence_matches_v1, action_application_basis_matches_state_v1, - action_batch_composition_digest_from_receipts_v1, action_preparation_identity_matches_v1, - admit_action_invocation_v1, admit_invocation_v1, admit_package_v1, - commit_prepared_to_state, decode_invocation_route_v1, + action_batch_composition_digest_from_receipts_v1, action_batch_patch_from_preparations_v1, + action_preparation_identity_matches_v1, admit_action_invocation_v1, admit_invocation_v1, + admit_package_v1, commit_prepared_to_state, decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, inspect_action_invocation_v1, install_recovered_v1, installed_from_admitted, not_committed_basis_changed, not_committed_evaluation_authority_mismatch, not_committed_installation_unavailable, @@ -4101,8 +4101,69 @@ fn validate_recovered_echo_operation_parent_states( .or_default() .push((*ingress_id, *submission_id)); } - for (_, mut members) in action_tick_members { + for ((head_key, worldline_tick_after, _, _), mut members) in action_tick_members { members.sort_by_key(|(ingress_id, _)| *ingress_id); + let tick_before = worldline_tick_after + .as_u64() + .checked_sub(1) + .map(crate::WorldlineTick::from_raw) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has an impossible parent coordinate", + })?; + let transition = entries.get(&(head_key.worldline_id, tick_before)).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has no recovered state transition", + }, + )?; + let policy_id = transition + .patch + .as_ref() + .map(crate::WorldlineTickPatchV1::policy_id) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has no retained replay patch", + })?; + let ordered_rule_ids = members + .iter() + .map(|(_, submission_id)| match outcomes.get(submission_id) { + Some(EchoOperationActionOutcomeV1::Committed(receipt)) => { + Ok(receipt.installed_operation_id().as_hash()) + } + Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) => { + Ok(obstruction.installed_operation_id().as_hash()) + } + Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) => { + Ok(conflict.installed_operation_id.as_hash()) + } + None => Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no recovered outcome", + }), + }) + .collect::, _>>()?; + let applied_preparations = members + .iter() + .filter_map(|(_, submission_id)| { + if matches!( + outcomes.get(submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + ) { + reconstructed_preparations + .get(submission_id) + .map(Box::as_ref) + } else { + None + } + }) + .collect::>(); + let reconstructed_patch = action_batch_patch_from_preparations_v1( + policy_id, + &ordered_rule_ids, + &applied_preparations, + ); + if reconstructed_patch.digest() != transition.expected.patch_digest { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick patch omits or alters a reconstructed applied member", + }); + } for (index, (_, submission_id)) in members.iter().enumerate() { let Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) = outcomes.get(submission_id) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 91980130..7b7294eb 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2954,6 +2954,28 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { forged_composition.validate_echo_operation_action_outcomes_for_test(), Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) )); + let mut omitted_member_patch = runtime_wal + .recover_read_only() + .expect("the honest composite Tick recovers for membership mutation"); + let forged_patch_digest = digest("composite-patch-omits-an-applied-member"); + let transition = omitted_member_patch + .provenance_entries + .iter_mut() + .find(|entry| entry.expected.commit_hash == steps[0].commit_hash) + .expect("the recovered scheduler transition remains available"); + transition.expected.patch_digest = forged_patch_digest; + transition + .patch + .as_mut() + .expect("the scheduler transition retains its replay patch") + .patch_digest = forged_patch_digest; + assert!(matches!( + omitted_member_patch.validate_echo_operation_parent_states_for_test( + host.runtime(), + host.provenance() + ), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); assert!( adversarial .echo_operation_action_outcomes From e7090e8443cdb57215ada3a0815a3ef2d3d9c7ef Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:57:06 -0700 Subject: [PATCH 32/64] Fix: initialize footprint fixture directly --- crates/warp-core/src/echo_operation.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 5dc99d07..e7cdee77 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -5910,9 +5910,10 @@ mod tests { let mut footprint = Footprint::default(); footprint.b_out.insert(node.warp_id, 9); footprints.push(footprint); - let mut footprint = Footprint::default(); - footprint.factor_mask = 1; - footprints.push(footprint); + footprints.push(Footprint { + factor_mask: 1, + ..Footprint::default() + }); let digests = footprints.iter().map(footprint_digest).collect::>(); for (index, digest) in digests.iter().enumerate() { From 63213a0fcbc46afbbf506683f2ebde3321b25c23 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 09:59:00 -0700 Subject: [PATCH 33/64] Fix: reuse descended fixture state --- crates/warp-core/src/echo_operation.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index e7cdee77..cb347483 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -6025,7 +6025,8 @@ mod tests { }, child_store, ); - let state = WorldlineState::new(warp_state, root).expect("the descended fixture is lawful"); + let mut state = + WorldlineState::new(warp_state, root).expect("the descended fixture is lawful"); let operation_coordinate = "echo.fixture.DescendedCreateIfAbsent.v1"; let authority_profile = digest(40); @@ -6138,8 +6139,7 @@ mod tests { "each portal pointer read must be charged as a bounded evaluator step" ); - let mut budget_limited_state = state.clone(); - budget_limited_state + state .warp_state .store_mut(&root_warp) .expect("the fixture retains its root store") @@ -6154,7 +6154,7 @@ mod tests { writer_head, WorldlineTick::ZERO, None, - budget_limited_state.state_root(), + state.state_root(), digest(51), echo_operation_anchored_node_absent_application_basis_v1(target), ); @@ -6179,7 +6179,7 @@ mod tests { .to_canonical_bytes() .expect("budget-limited invocation encodes"), budget_limited_basis, - &budget_limited_state, + &state, evaluation_authority.clone(), ) .expect("the budget-limited invocation admits"); @@ -6187,7 +6187,7 @@ mod tests { Some(&installed), admitted, budget_limited_basis, - &budget_limited_state, + &state, crate::POLICY_ID_NO_POLICY_V0, &evaluation_authority, ) else { From 20acffc854b9b1a58aaa5520584678dc30dba78d Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 10:30:19 -0700 Subject: [PATCH 34/64] Fix: requeue Actions after scheduler WAL rollback --- CHANGELOG.md | 5 ++- crates/warp-core/src/trusted_runtime_host.rs | 6 ++++ .../executable_operation_pipeline_tests.rs | 36 ++++++++++++------- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 678d9b76..9e959028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,10 @@ One scheduler WAL transaction retains exactly one batched Tick decision record, then each Action's receipt correlation and typed outcome in canonical order, followed by exactly one replayable state delta. The decided Tick is - durable before state, frontier, receipt, or outcome publication. Fresh-host + durable before state, frontier, receipt, or outcome publication. Recovery + and same-host retry both preserve an accepted Action when Tick-WAL + persistence fails; runtime rollback also rolls back the corresponding + admission cache so the Action re-enters scheduler admission. Fresh-host recovery validates every outcome against its exact envelope, admission, invocation, installed operation, causal coordinate, evaluation basis, reconstructed preparation and actual footprint, Tick entry, composite diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index c22aba82..bf23703d 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -1810,11 +1810,13 @@ impl TrustedRuntimeHost { let runtime_before = self.runtime.clone(); let provenance_before = self.provenance.clone(); let action_outcomes_before = self.echo_operation_action_outcomes.clone(); + let admitted_actions_before = self.admitted_echo_operation_actions.clone(); let admitted_actions = match self.admit_pending_echo_operation_actions_v1() { Ok(admitted) => admitted, Err(error) => { self.runtime = runtime_before; self.provenance = provenance_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(error); } }; @@ -1850,6 +1852,7 @@ impl TrustedRuntimeHost { self.runtime = runtime_before; self.provenance = provenance_before; self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(error.into()); } }; @@ -1860,6 +1863,7 @@ impl TrustedRuntimeHost { self.runtime = runtime_before; self.provenance = provenance_before; self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(error); } }; @@ -1887,6 +1891,7 @@ impl TrustedRuntimeHost { self.runtime = runtime_before; self.provenance = provenance_before; self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(TrustedRuntimeWalError::FilesystemAtomicBatchUnsupported { transaction_kind: WalTransactionKind::SchedulerTick, transaction_count: tick_wal_groups.len(), @@ -1931,6 +1936,7 @@ impl TrustedRuntimeHost { self.runtime = runtime_before; self.provenance = provenance_before; self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(error.into()); } } diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 7b7294eb..92284721 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3730,7 +3730,7 @@ fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { #[cfg(feature = "host_test")] #[test] -fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { +fn scheduler_wal_failure_rolls_back_and_requeues_action() { let wal_dir = TempWalDir::new(); let submission_id; let head_key; @@ -3793,26 +3793,36 @@ fn scheduler_wal_failure_publishes_no_action_state_or_receipt() { Bytes::from_static(b"before"), ))) ); + + host.tick_once() + .expect("the same host retries the still-pending Action"); + assert!(matches!( + host.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + assert_eq!( + host.runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the retried worldline remains available") + .state() + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"must-rollback"), + ))) + ); } let (mut recovered, _, _) = fixture_host(); recovered .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) - .expect("recovery discards the uncommitted Tick tail"); + .expect("recovery discards the failed Tick tail and restores the retry"); assert!(matches!( recovered.runtime().observe_intent_outcome(&submission_id), - warp_core::IntentOutcomeObservation::Pending { - ticketed_ingress_id: None, - .. - } + warp_core::IntentOutcomeObservation::Decided { .. } )); - assert!(recovered - .echo_operation_action_outcome_v1(&submission_id) - .is_none()); - recovered.install_echo_operation_action_admission_policy_v1(invocation_policy()); - recovered - .tick_once() - .expect("the recovered pending Action remains executable"); assert!(matches!( recovered.echo_operation_action_outcome_v1(&submission_id), Some(EchoOperationActionOutcomeV1::Committed(_)) From 5ee48fe0c098037b30d25da678ae5d3656fca1df Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 17:32:23 -0700 Subject: [PATCH 35/64] Fix: reconstruct recovered Action obstructions --- CHANGELOG.md | 3 + crates/warp-core/src/echo_operation.rs | 119 +++++++++-- crates/warp-core/src/trusted_runtime_host.rs | 184 +++++++++++++++++- .../executable_operation_pipeline_tests.rs | 15 ++ 4 files changed, 297 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e959028..032859bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ invocation, installed operation, causal coordinate, evaluation basis, reconstructed preparation and actual footprint, Tick entry, composite consequence, exact reconstructed aggregate patch membership, and state root. + Typed obstruction records retain their invocation-admission policy and + budget ceiling; recovery reproduces bounded evaluation and the complete + scheduler composition before accepting an obstruction kind. A composite receipt cannot validate outside its complete Action-batch context. Legacy operation recovery-index roots remain byte-compatible when no Action outcome exists. Direct operation diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index cb347483..280db04d 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -84,7 +84,7 @@ const ACTION_BATCH_RULE_PACK_DOMAIN: &[u8] = b"echo:operation-action-batch-rule- const ACTION_BATCH_PLAN_DOMAIN: &[u8] = b"echo:operation-action-batch-plan:v1\0"; const ACTION_BATCH_REWRITES_DOMAIN: &[u8] = b"echo:operation-action-batch-rewrites:v1\0"; const ACTION_BATCH_COMPOSITION_DOMAIN: &[u8] = b"echo:operation-action-batch-composition:v1\0"; -const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT002"; +const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT003"; pub(crate) const ACTION_BATCH_CANDIDATE_LIMIT_V1: usize = 64; const ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1: usize = ACTION_BATCH_CANDIDATE_LIMIT_V1 * (ACTION_BATCH_CANDIDATE_LIMIT_V1 - 1) / 2; @@ -2604,7 +2604,7 @@ pub(crate) fn action_preparation_identity_matches_v1( ) == expected_preparation_id } -pub(crate) fn reconstruct_action_preparation_v1( +pub(crate) fn reconstruct_action_evaluation_v1( installed: &InstalledEchoOperationV1, canonical_invocation_bytes: &[u8], maximum_budget: EchoOperationBudgetV1, @@ -2612,7 +2612,7 @@ pub(crate) fn reconstruct_action_preparation_v1( expected_admission_id: EchoOperationInvocationAdmissionIdV1, state: &WorldlineState, policy_id: u32, -) -> Option> { +) -> Option { if !action_admission_evidence_matches_v1( installed, canonical_invocation_bytes, @@ -2640,14 +2640,34 @@ pub(crate) fn reconstruct_action_preparation_v1( if admitted.admission_id() != expected_admission_id { return None; } - match prepare_operation_v1( + Some(prepare_operation_v1( Some(installed), admitted, invocation.evaluation_basis, state, policy_id, &authority, - ) { + )) +} + +pub(crate) fn reconstruct_action_preparation_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + maximum_budget: EchoOperationBudgetV1, + expected_policy_id: Hash, + expected_admission_id: EchoOperationInvocationAdmissionIdV1, + state: &WorldlineState, + policy_id: u32, +) -> Option> { + match reconstruct_action_evaluation_v1( + installed, + canonical_invocation_bytes, + maximum_budget, + expected_policy_id, + expected_admission_id, + state, + policy_id, + )? { EchoOperationPreparationV1::Prepared(prepared) => Some(prepared), EchoOperationPreparationV1::Obstructed(_) => None, } @@ -2718,7 +2738,11 @@ pub(crate) fn runtime_basis_obstruction( kind: EchoOperationObstructionKindV1::BasisChanged, package_id: admitted.invocation.package_id, installed_operation_id: admitted.installed_operation_id, - invocation_admission_id: admitted.admission_id, + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: admitted.admission_policy_id, + maximum_budget: admitted.admission_policy.maximum_delegated_budget, + admission_id: admitted.admission_id, + }), invocation_id: admitted.invocation_id, evaluation_basis_id: admitted.invocation.evaluation_basis.identity(), }) @@ -2755,13 +2779,21 @@ pub enum EchoOperationObstructionKindV1 { ReplacementTooLarge, } +/// Retained runtime policy evidence needed to reproduce one obstruction. +#[derive(Clone, Debug, PartialEq, Eq)] +struct EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: Hash, + maximum_budget: EchoOperationBudgetV1, + admission_id: EchoOperationInvocationAdmissionIdV1, +} + /// One typed obstruction. Obstruction never carries a parent-visible patch. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EchoOperationObstructionV1 { kind: EchoOperationObstructionKindV1, package_id: EchoOperationPackageIdV1, installed_operation_id: InstalledEchoOperationIdV1, - invocation_admission_id: EchoOperationInvocationAdmissionIdV1, + invocation_admission: Box, invocation_id: EchoOperationInvocationIdV1, evaluation_basis_id: EchoOperationEvaluationBasisIdV1, } @@ -2800,7 +2832,15 @@ impl EchoOperationObstructionV1 { /// Returns the exact runtime admission that authorized this evaluation attempt. #[must_use] pub const fn invocation_admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { - self.invocation_admission_id + self.invocation_admission.admission_id + } + + pub(crate) const fn invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission.policy_id + } + + pub(crate) const fn invocation_admission_maximum_budget(&self) -> EchoOperationBudgetV1 { + self.invocation_admission.maximum_budget } /// Returns the identity of this typed no-parent-patch obstruction. @@ -2810,7 +2850,7 @@ impl EchoOperationObstructionV1 { hasher.update(OBSTRUCTION_ID_DOMAIN); hasher.update(&self.package_id.as_hash()); hasher.update(&self.installed_operation_id.as_hash()); - hasher.update(&self.invocation_admission_id.as_hash()); + hasher.update(&self.invocation_admission.admission_id.as_hash()); hasher.update(&self.invocation_id.as_hash()); hasher.update(&self.evaluation_basis_id.as_hash()); hasher.update(&[obstruction_kind_code(self.kind)]); @@ -2945,6 +2985,16 @@ pub enum EchoOperationActionOutcomeV1 { } impl EchoOperationActionOutcomeV1 { + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_obstruction_kind_for_test( + &mut self, + kind: EchoOperationObstructionKindV1, + ) { + if let Self::Obstructed(obstruction) = self { + obstruction.kind = kind; + } + } + #[cfg(any(test, feature = "host_test"))] pub(crate) fn replace_conflict_preparation_id_for_test(&mut self, digest: Hash) { if let Self::RejectedFootprintConflict(conflict) = self { @@ -2983,7 +3033,29 @@ pub(crate) fn retain_action_outcome_v1( out.push(obstruction_kind_code(obstruction.kind)); out.extend_from_slice(&obstruction.package_id.as_hash()); out.extend_from_slice(&obstruction.installed_operation_id.as_hash()); - out.extend_from_slice(&obstruction.invocation_admission_id.as_hash()); + out.extend_from_slice(&obstruction.invocation_admission.policy_id); + out.extend_from_slice( + &obstruction + .invocation_admission + .maximum_budget + .steps + .to_le_bytes(), + ); + out.extend_from_slice( + &obstruction + .invocation_admission + .maximum_budget + .read_bytes + .to_le_bytes(), + ); + out.extend_from_slice( + &obstruction + .invocation_admission + .maximum_budget + .write_bytes + .to_le_bytes(), + ); + out.extend_from_slice(&obstruction.invocation_admission.admission_id.as_hash()); out.extend_from_slice(&obstruction.invocation_id.as_hash()); out.extend_from_slice(&obstruction.evaluation_basis_id.as_hash()); } @@ -3075,9 +3147,18 @@ pub(crate) fn recover_action_outcome_v1( bytes, &mut offset, )?), - invocation_admission_id: EchoOperationInvocationAdmissionIdV1( - read_action_outcome_hash(bytes, &mut offset)?, - ), + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: read_action_outcome_hash(bytes, &mut offset)?, + maximum_budget: EchoOperationBudgetV1 { + steps: read_action_outcome_u64(bytes, &mut offset)?, + read_bytes: read_action_outcome_u64(bytes, &mut offset)?, + write_bytes: read_action_outcome_u64(bytes, &mut offset)?, + }, + admission_id: EchoOperationInvocationAdmissionIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + }), invocation_id: EchoOperationInvocationIdV1(read_action_outcome_hash( bytes, &mut offset, @@ -3370,7 +3451,11 @@ pub(crate) fn prepare_operation_v1( kind, package_id, installed_operation_id: admitted.installed_operation_id, - invocation_admission_id: admitted.admission_id, + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: admitted.admission_policy_id, + maximum_budget: admitted.admission_policy.maximum_delegated_budget, + admission_id: admitted.admission_id, + }), invocation_id, evaluation_basis_id: admitted.invocation.evaluation_basis.identity(), }) @@ -4497,7 +4582,11 @@ fn scheduler_composition_budget_obstruction_v1( kind: EchoOperationObstructionKindV1::BudgetExceeded, package_id: prepared.package_id(), installed_operation_id: prepared.installed_operation_id(), - invocation_admission_id: prepared.invocation_admission_id(), + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: prepared.invocation_admission_policy_id(), + maximum_budget: prepared.invocation_admission_maximum_budget(), + admission_id: prepared.invocation_admission_id(), + }), invocation_id: prepared.invocation_id(), evaluation_basis_id: prepared.evaluation_basis().identity(), } diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index bf23703d..a123c5cd 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -45,14 +45,16 @@ use crate::{ action_admission_evidence_matches_v1, action_application_basis_matches_state_v1, action_batch_composition_digest_from_receipts_v1, action_batch_patch_from_preparations_v1, action_preparation_identity_matches_v1, admit_action_invocation_v1, admit_invocation_v1, - admit_package_v1, commit_prepared_to_state, decode_invocation_route_v1, - echo_operation_action_invocation_bytes_v1, inspect_action_invocation_v1, - install_recovered_v1, installed_from_admitted, not_committed_basis_changed, - not_committed_evaluation_authority_mismatch, not_committed_installation_unavailable, - operation_descent_stack, prepare_operation_v1, reconstruct_action_preparation_v1, + admit_package_v1, commit_prepared_to_state, commit_scheduler_action_batch_to_state_v1, + decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, + inspect_action_invocation_v1, install_recovered_v1, installed_from_admitted, + not_committed_basis_changed, not_committed_evaluation_authority_mismatch, + not_committed_installation_unavailable, operation_descent_stack, prepare_operation_v1, + reconstruct_action_evaluation_v1, reconstruct_action_preparation_v1, recover_action_outcome_v1, recover_committed_execution_receipt_v1, recover_installation_v1, retain_action_outcome_v1, retain_committed_execution_v1, retain_installation_v1, validate_receipt_installation_v1, EchoOperationEvaluationAuthorityV1, + SchedulerEchoOperationCandidateV1, }, provider_contract::admit_provider_contract_package_v1, AdmittedEchoOperationInvocationV1, AdmittedExecutableOperationPackageV1, @@ -499,6 +501,22 @@ impl TrustedRuntimeWalRecovery { .remove(&submission_id); } + /// Replaces one retained obstruction kind for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_obstruction_kind_for_test( + &mut self, + submission_id: Hash, + kind: crate::EchoOperationObstructionKindV1, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(candidate, _, _)| *candidate == submission_id) + { + outcome.replace_obstruction_kind_for_test(kind); + } + } + /// Replaces one rejected preparation identity for adversarial tests. #[cfg(any(test, feature = "host_test"))] pub fn replace_echo_operation_action_conflict_preparation_for_test( @@ -3945,6 +3963,8 @@ fn validate_recovered_echo_operation_parent_states( .map(|(submission_id, _, outcome)| (*submission_id, outcome)) .collect::>(); let mut reconstructed_preparations = BTreeMap::new(); + let mut reconstructed_evaluations = BTreeMap::new(); + let mut action_scopes = BTreeMap::new(); let mut action_tick_members = BTreeMap::< ( crate::WriterHeadKey, @@ -3974,6 +3994,7 @@ fn validate_recovered_echo_operation_parent_states( detail: "Action outcome invocation cannot be inspected", } })?; + action_scopes.insert(*submission_id, invocation.scope); let installed = installations.get(&invocation.package_id).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action outcome has no recovered installation", @@ -4061,6 +4082,10 @@ fn validate_recovered_echo_operation_parent_states( "committed Action evidence disagrees with reconstructed preparation", }); } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Prepared(prepared.clone()), + ); Some(prepared) } EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { @@ -4090,9 +4115,66 @@ fn validate_recovered_echo_operation_parent_states( detail: "conflict evidence disagrees with reconstructed preparation", }); } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Prepared(prepared.clone()), + ); Some(prepared) } - EchoOperationActionOutcomeV1::Obstructed(_) => None, + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + if basis_changed { + if !action_admission_evidence_matches_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + ) { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "basis obstruction has invalid invocation-admission evidence", + }); + } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Obstructed(obstruction.clone()), + ); + None + } else { + let evaluation = reconstruct_action_evaluation_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + &basis_state, + policy_id, + ) + .ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "obstructed Action evaluation cannot be reconstructed", + }, + )?; + match &evaluation { + EchoOperationPreparationV1::Obstructed(reconstructed) + if reconstructed == obstruction => {} + EchoOperationPreparationV1::Prepared(_) + if obstruction.kind() + == crate::EchoOperationObstructionKindV1::BudgetExceeded => {} + _ => { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: + "obstruction evidence disagrees with reconstructed evaluation", + }) + } + } + let prepared = match &evaluation { + EchoOperationPreparationV1::Prepared(prepared) => Some(prepared.clone()), + EchoOperationPreparationV1::Obstructed(_) => None, + }; + reconstructed_evaluations.insert(*submission_id, evaluation); + prepared + } + } }; if let Some(prepared) = reconstructed { reconstructed_preparations.insert(*submission_id, prepared); @@ -4107,7 +4189,9 @@ fn validate_recovered_echo_operation_parent_states( .or_default() .push((*ingress_id, *submission_id)); } - for ((head_key, worldline_tick_after, _, _), mut members) in action_tick_members { + for ((head_key, worldline_tick_after, commit_global_tick, _), mut members) in + action_tick_members + { members.sort_by_key(|(ingress_id, _)| *ingress_id); let tick_before = worldline_tick_after .as_u64() @@ -4128,6 +4212,75 @@ fn validate_recovered_echo_operation_parent_states( .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action Tick has no retained replay patch", })?; + let frontier = runtime.worldlines().get(&head_key.worldline_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick names an unavailable recovery worldline", + }, + )?; + let mut reconstructed_state = recovered_provenance + .replay_worldline_state_at(head_key.worldline_id, frontier.state(), tick_before) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick parent state cannot be reconstructed", + })?; + let candidates = members + .iter() + .map( + |(ingress_id, submission_id)| -> Result<_, TrustedRuntimeWalError> { + let outcome = outcomes.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no recovered outcome", + }, + )?; + let rule_id = match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + receipt.installed_operation_id().as_hash() + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + obstruction.installed_operation_id().as_hash() + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + conflict.installed_operation_id.as_hash() + } + }; + Ok(SchedulerEchoOperationCandidateV1 { + submission_id: *submission_id, + ingress_id: *ingress_id, + scope: *action_scopes.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no reconstructed scope", + }, + )?, + rule_id, + preparation: reconstructed_evaluations + .get(submission_id) + .cloned() + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no reconstructed evaluation", + })?, + }) + }, + ) + .collect::, _>>()?; + let reconstructed_batch = commit_scheduler_action_batch_to_state_v1( + candidates, + &mut reconstructed_state, + commit_global_tick, + policy_id, + ) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick composition cannot be reconstructed", + })?; + let reconstructed_outcomes = reconstructed_batch + .outcomes + .into_iter() + .collect::>(); + if members.iter().any(|(_, submission_id)| { + reconstructed_outcomes.get(submission_id) != outcomes.get(submission_id).copied() + }) { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome disagrees with reconstructed Tick composition", + }); + } let ordered_rule_ids = members .iter() .map(|(_, submission_id)| match outcomes.get(submission_id) { @@ -4690,6 +4843,13 @@ fn validate_recovered_echo_operation_action_outcomes( EchoOperationActionOutcomeV1::Obstructed(obstruction) => { if obstruction.installed_operation_id() != installed.installed_operation_id() || obstruction.package_id() != invocation.package_id + || !action_admission_evidence_matches_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + ) || obstruction.invocation_id() != invocation.invocation_id || obstruction.evaluation_basis_id() != invocation.evaluation_basis.identity() { @@ -6951,11 +7111,17 @@ mod tests { receipt_ref: receipt.receipt_ref, causal_parent_receipts: Vec::new(), }; - let mut action_outcome = b"EOACT002".to_vec(); + let mut action_outcome = b"EOACT003".to_vec(); action_outcome.extend_from_slice(&receipt.receipt_ref.submission_id); action_outcome.extend_from_slice(&[45; 32]); action_outcome.extend_from_slice(&[2, 1]); - for byte in 46..=50 { + for byte in 46..=48 { + action_outcome.extend_from_slice(&[byte; 32]); + } + action_outcome.extend_from_slice(&1_u64.to_le_bytes()); + action_outcome.extend_from_slice(&2_u64.to_le_bytes()); + action_outcome.extend_from_slice(&3_u64.to_le_bytes()); + for byte in 49..=51 { action_outcome.extend_from_slice(&[byte; 32]); } diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 92284721..228ff335 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3317,6 +3317,21 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { .2 .ops() .is_empty()); + + let mut adversarial = host + .runtime_wal() + .expect("the obstruction fixture retains its WAL") + .recover_read_only() + .expect("the honest obstruction WAL recovers"); + adversarial.replace_echo_operation_action_obstruction_kind_for_test( + submission_id, + EchoOperationObstructionKindV1::BudgetExceeded, + ); + assert!(matches!( + adversarial + .validate_echo_operation_parent_states_for_test(host.runtime(), host.provenance()), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); } let (mut recovered, head_key, node) = fixture_host(); From 909dec837b1805bf628f1eff97b640e2e57d2a4f Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 17:34:30 -0700 Subject: [PATCH 36/64] Fix: gate obstruction mutation harness --- .../executable_operation_pipeline_tests.rs | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 228ff335..f67456ea 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3318,20 +3318,25 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { .ops() .is_empty()); - let mut adversarial = host - .runtime_wal() - .expect("the obstruction fixture retains its WAL") - .recover_read_only() - .expect("the honest obstruction WAL recovers"); - adversarial.replace_echo_operation_action_obstruction_kind_for_test( - submission_id, - EchoOperationObstructionKindV1::BudgetExceeded, - ); - assert!(matches!( - adversarial - .validate_echo_operation_parent_states_for_test(host.runtime(), host.provenance()), - Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) - )); + #[cfg(feature = "host_test")] + { + let mut adversarial = host + .runtime_wal() + .expect("the obstruction fixture retains its WAL") + .recover_read_only() + .expect("the honest obstruction WAL recovers"); + adversarial.replace_echo_operation_action_obstruction_kind_for_test( + submission_id, + EchoOperationObstructionKindV1::BudgetExceeded, + ); + assert!(matches!( + adversarial.validate_echo_operation_parent_states_for_test( + host.runtime(), + host.provenance() + ), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); + } } let (mut recovered, head_key, node) = fixture_host(); From 6c5268f3ab627c4b20075d174c09367fda7b425e Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 17:40:15 -0700 Subject: [PATCH 37/64] Fix: reject split scheduler Tick transactions --- CHANGELOG.md | 7 +++- crates/warp-core/src/trusted_runtime_host.rs | 42 ++++++++++++------- .../executable_operation_pipeline_tests.rs | 4 ++ 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 032859bd..e02411df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,11 @@ and composition-budget obstructions contribute no operations. One scheduler WAL transaction retains exactly one batched Tick decision record, then each Action's receipt correlation and typed outcome in canonical - order, followed by exactly one replayable state delta. The decided Tick is - durable before state, frontier, receipt, or outcome publication. Recovery + order, followed by exactly one replayable state delta. Recovery rejects every + second transaction claiming the same state-transition coordinate, including + a byte-identical delta, so a Tick cannot be reconstructed from split WAL + fragments. The decided Tick is durable before state, frontier, receipt, or + outcome publication. Recovery and same-host retry both preserve an accepted Action when Tick-WAL persistence fails; runtime rollback also rolls back the corresponding admission cache so the Action re-enters scheduler admission. Fresh-host diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index a123c5cd..d0887792 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -2356,6 +2356,26 @@ impl TrustedRuntimeWal { self.store.cloned_in_memory_store() } + /// Re-runs state-delta recovery after repeating the last scheduler + /// transaction, for adversarial transaction-atomicity tests. + #[cfg(any(test, feature = "host_test"))] + pub fn recover_with_repeated_scheduler_tick_for_test( + &self, + ) -> Result<(), TrustedRuntimeWalError> { + let mut report = self.store.recover_read_only()?; + let repeated = report + .transactions + .iter() + .rev() + .find(|transaction| { + transaction.commit.transaction_kind == WalTransactionKind::SchedulerTick + }) + .cloned() + .ok_or_else(missing_trusted_runtime_record)?; + report.transactions.push(repeated); + recover_runtime_state_delta_material(&report).map(|_| ()) + } + /// Recovers submission and receipt indexes from committed WAL transactions /// without scheduler callbacks. pub fn recover_read_only(&self) -> Result { @@ -4543,10 +4563,7 @@ fn recover_runtime_state_delta_material( let (_, state_delta, _) = operation_tick_records_from_transaction(transaction)?; let entry = state_delta.provenance_entry().clone(); let coordinate = (entry.worldline_id, entry.worldline_tick); - if entries_by_coordinate - .get(&coordinate) - .is_some_and(|existing| existing != &entry) - { + if entries_by_coordinate.contains_key(&coordinate) { return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { worldline_id: entry.worldline_id, worldline_tick: entry.worldline_tick, @@ -4583,6 +4600,13 @@ fn recover_runtime_state_delta_material( let head_key = entry .head_key .ok_or(RetainedProvenanceError::MissingHeadKey)?; + let coordinate = (entry.worldline_id, entry.worldline_tick); + if entries_by_coordinate.contains_key(&coordinate) { + return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { + worldline_id: entry.worldline_id, + worldline_tick: entry.worldline_tick, + }); + } for (receipt, wal_correlation, action_outcome) in records { let expected_receipt_ref = crate::CausalTickReceiptRef { worldline_id: entry.worldline_id, @@ -4648,16 +4672,6 @@ fn recover_runtime_state_delta_material( action_installations_before_tick.insert(submission_id, installed_packages.clone()); } } - let coordinate = (entry.worldline_id, entry.worldline_tick); - if entries_by_coordinate - .get(&coordinate) - .is_some_and(|existing| existing != &entry) - { - return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { - worldline_id: entry.worldline_id, - worldline_tick: entry.worldline_tick, - }); - } entries_by_coordinate.insert(coordinate, entry); } let mut entries = entries_by_coordinate.into_values().collect::>(); diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index f67456ea..0030540b 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2902,6 +2902,10 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { #[cfg(feature = "host_test")] { + assert!(matches!( + runtime_wal.recover_with_repeated_scheduler_tick_for_test(), + Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { .. }) + )); let mut adversarial = runtime_wal .recover_read_only() .expect("the honest composite Tick recovers"); From 6f8944915fbbf98d8562ca4b19cb9e7df9fbbbd6 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 19:41:35 -0700 Subject: [PATCH 38/64] Fix: require durable ACK for Action submission --- CHANGELOG.md | 2 ++ crates/warp-core/src/contract_obstruction.rs | 3 +- crates/warp-core/src/coordinator.rs | 14 +++++++- crates/warp-core/src/trusted_runtime_host.rs | 3 ++ .../executable_operation_pipeline_tests.rs | 36 +++++++------------ 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e02411df..e862a4a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ constructing a Tick (ADR 0025). Accepted pre-Tick Actions recover as pending work. Runtime-owned admission uses a bounded pending index and cache; unavailable packages are quarantined without poisoning unrelated work. + A WAL-enabled app surface rejects executable Actions submitted outside the + durable acknowledgement boundary before witnessed intake can mutate. Scheduler selection admits at most 64 executable Actions per Tick, leaving excess work pending, and meters footprint comparisons, blocker evidence, and aggregate operations during composition. Two independent Actions can share diff --git a/crates/warp-core/src/contract_obstruction.rs b/crates/warp-core/src/contract_obstruction.rs index 206c23b8..75fa6d99 100644 --- a/crates/warp-core/src/contract_obstruction.rs +++ b/crates/warp-core/src/contract_obstruction.rs @@ -230,7 +230,8 @@ impl ContractObstruction { ), #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] RuntimeError::EchoOperationCommit(_) - | RuntimeError::EchoOperationActionAdmissionMissing(_) => { + | RuntimeError::EchoOperationActionAdmissionMissing(_) + | RuntimeError::EchoOperationActionRequiresRuntimeWalAck => { Self::runtime_fault(ContractObstructionSubject::Unspecified) } RuntimeError::SchedulerFaultGenerationOverflow diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 219ea400..ec9141ad 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -138,6 +138,13 @@ pub enum RuntimeError { /// Ordinary submission attempted to claim the contract-inverse target role. #[error("contract inverse target parent requires contract inverse admission")] ContractInverseTargetRequiresContractAdmission, + /// A WAL-enabled host received an executable-operation Action through the + /// non-durable app submission method. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[error( + "executable-operation Actions on a WAL-enabled host require submit_intent_with_runtime_wal_ack" + )] + EchoOperationActionRequiresRuntimeWalAck, /// A commit against a worldline frontier failed. #[error(transparent)] Engine(#[from] EngineError), @@ -3281,7 +3288,8 @@ fn scheduler_fault_scope_for_error( SchedulerFaultScope::Runtime } #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] - RuntimeError::EchoOperationActionAdmissionMissing(_) => SchedulerFaultScope::Runtime, + RuntimeError::EchoOperationActionAdmissionMissing(_) + | RuntimeError::EchoOperationActionRequiresRuntimeWalAck => SchedulerFaultScope::Runtime, } } @@ -3932,6 +3940,10 @@ fn scheduler_error_cause_digest(err: &RuntimeError) -> Hash { hasher.update(b"echo-operation-action-admission-missing"); hasher.update(ingress_id); } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationActionRequiresRuntimeWalAck => { + hasher.update(b"echo-operation-action-requires-runtime-wal-ack"); + } RuntimeError::FrontierTickOverflow(worldline_id) => { hasher.update(b"frontier-tick-overflow"); hasher.update(worldline_id.as_bytes()); diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index d0887792..61994169 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -5502,6 +5502,9 @@ impl TrustedRuntimeApp<'_> { ) -> Result { let is_echo_operation_action = echo_operation_action_invocation_bytes_v1(&envelope).is_some(); + if is_echo_operation_action && self.host.runtime_wal.is_some() { + return Err(RuntimeError::EchoOperationActionRequiresRuntimeWalAck); + } let handle = self.host.runtime.submit_app_intent(envelope)?; if self.host.runtime_wal.is_none() { self.host.track_pending_echo_operation_action_v1( diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 0030540b..e2079c20 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -18,6 +18,8 @@ use warp_core::causal_wal::{ }; #[cfg(feature = "host_test")] use warp_core::causal_wal::{FilesystemWalFaultPlan, FilesystemWalFaultTarget, WalTickDecision}; +#[cfg(feature = "host_test")] +use warp_core::EchoOperationCommitErrorV1; use warp_core::{ make_head_id, make_node_id, make_type_id, make_warp_id, AtomPayload, AttachmentValue, EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorKindV1, @@ -27,13 +29,11 @@ use warp_core::{ EchoOperationInvocationV1, EchoOperationObstructionKindV1, EchoOperationPreparationV1, EchoOperationProgramV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, EngineBuilder, ExecutableOperationPackageV1, GraphStore, InboxPolicy, IngressTarget, - InstalledEchoOperationV1, NodeKey, NodeRecord, PlaybackMode, RuntimeWalActivationGap, - SchedulerKind, TrustedRuntimeHost, TrustedRuntimeHostError, TrustedRuntimeWalConfig, - TrustedRuntimeWalError, WorldlineId, WorldlineRuntime, WorldlineState, WriterHead, - WriterHeadKey, + InstalledEchoOperationV1, NodeKey, NodeRecord, PlaybackMode, RuntimeError, + RuntimeWalActivationGap, SchedulerKind, TrustedRuntimeHost, TrustedRuntimeHostError, + TrustedRuntimeWalConfig, TrustedRuntimeWalError, WorldlineId, WorldlineRuntime, WorldlineState, + WriterHead, WriterHeadKey, }; -#[cfg(feature = "host_test")] -use warp_core::{EchoOperationCommitErrorV1, RuntimeError}; const OPERATION_COORDINATE: &str = "echo.fixture.SetAnchoredAtom.v1"; const CREATE_OPERATION_COORDINATE: &str = "echo.fixture.CreateAnchoredAtomIfAbsent.v1"; @@ -3198,7 +3198,7 @@ fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { } #[test] -fn wal_enabled_host_does_not_auto_admit_non_durable_action_submission() { +fn wal_enabled_host_rejects_action_submission_without_durable_ack() { let (mut host, head_key, node) = fixture_host(); host.enable_in_memory_runtime_wal() .expect("the non-durable submission fixture WAL opens"); @@ -3217,26 +3217,16 @@ fn wal_enabled_host_does_not_auto_admit_non_durable_action_submission() { invocation, ) .expect("the invocation becomes an Action"); - let submission_id = host + let error = host .app() .submit_intent(envelope) - .expect("ordinary intake records process-local witnessed history") - .submission_id; - - let steps = host - .tick_once() - .expect("non-durable intake cannot enter the scheduler"); - assert!(steps.is_empty()); - assert!(host - .echo_operation_action_outcome_v1(&submission_id) - .is_none()); + .expect_err("a WAL-enabled Action must use the durable ACK boundary"); assert!(matches!( - host.runtime().observe_intent_outcome(&submission_id), - warp_core::IntentOutcomeObservation::Pending { - ticketed_ingress_id: None, - .. - } + error, + RuntimeError::EchoOperationActionRequiresRuntimeWalAck )); + + assert_eq!(host.runtime().pending_witnessed_submission_count(), 0); let recovery = host .runtime_wal() .expect("the runtime WAL remains enabled") From cc42aeeba109c37b05dd0b3882520f6181233dba Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 19:45:45 -0700 Subject: [PATCH 39/64] Fix: prevent execution category starvation --- CHANGELOG.md | 3 + crates/warp-core/src/coordinator.rs | 7 ++ crates/warp-core/src/head_inbox.rs | 91 ++++++++++++++++--- ...uler-owned-executable-operation-actions.md | 13 ++- docs/spec/scheduler-warp-core.md | 13 +-- 5 files changed, 104 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e862a4a4..ee2c3d85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ unavailable packages are quarantined without poisoning unrelated work. A WAL-enabled app surface rejects executable Actions submitted outside the durable acknowledgement boundary before witnessed intake can mutate. + Mixed executable and provider/native backlogs alternate by durable parent + worldline-Tick parity, preventing caller-controlled ingress hashes from + starving either execution category. Scheduler selection admits at most 64 executable Actions per Tick, leaving excess work pending, and meters footprint comparisons, blocker evidence, and aggregate operations during composition. Two independent Actions can share diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index ec9141ad..e5017706 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -4229,6 +4229,12 @@ impl SchedulerCoordinator { provenance.checkpoint_for(keys.iter().map(|key| key.worldline_id))?; for key in &keys { + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let partition_parent_tick = runtime + .worldlines + .get(&key.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))? + .frontier_tick(); let inbox = runtime .heads .inbox_mut(key) @@ -4238,6 +4244,7 @@ impl SchedulerCoordinator { inbox.admit_partitioned( crate::echo_operation::echo_operation_action_intent_kind_v1(), crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1, + partition_parent_tick, ) } else { inbox.admit() diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index 950c0f0b..d0b13ec4 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -794,24 +794,42 @@ impl HeadInbox { /// Admits one deterministic execution category without mixing it with /// other pending categories. /// - /// The lowest canonical ingress id chooses whether this batch contains the - /// supplied `partition_kind` or everything else. Entries from the other - /// category remain pending for a later Tick. Existing per-Tick limits still - /// bound the selected category. + /// When both categories are pending, `parent_worldline_tick` parity chooses + /// whether this batch contains the supplied `partition_kind` or everything + /// else. Using the durable parent coordinate means neither category can + /// starve and recovery does not depend on process-local state. When only one + /// category is pending, it proceeds immediately. Existing per-Tick limits + /// still bound the selected category. #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] pub(crate) fn admit_partitioned( &mut self, partition_kind: IntentKind, partition_limit: usize, + parent_worldline_tick: crate::WorldlineTick, ) -> Vec { - let Some(first) = self.pending.first_key_value().map(|(_, envelope)| envelope) else { + if self.pending.is_empty() { return Vec::new(); + } + let mut has_partition = false; + let mut has_other = false; + for envelope in self.pending.values() { + let in_partition = matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ); + has_partition |= in_partition; + has_other |= !in_partition; + if has_partition && has_other { + break; + } + } + let selected_partition = match (has_partition, has_other) { + (true, true) => parent_worldline_tick.as_u64().is_multiple_of(2), + (true, false) => true, + (false, true) => false, + (false, false) => return Vec::new(), }; - let selected_partition = matches!( - first.payload(), - IngressPayload::LocalIntent { intent_kind, .. } - if *intent_kind == partition_kind - ); let policy_limit = match self.policy { InboxPolicy::Budgeted { max_per_tick } => max_per_tick as usize, InboxPolicy::AcceptAll | InboxPolicy::KindFilter(_) => usize::MAX, @@ -984,7 +1002,8 @@ mod tests { assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); } - let first_batch = inbox.admit_partitioned(partition_kind, 2); + let first_tick = crate::WorldlineTick::from_raw(u64::from(!first_is_partition)); + let first_batch = inbox.admit_partitioned(partition_kind, 2, first_tick); assert_eq!(first_batch.len(), 2); assert!(first_batch.iter().all(|envelope| { matches!( @@ -995,7 +1014,10 @@ mod tests { })); assert_eq!(inbox.pending_count(), 2); - let second_batch = inbox.admit_partitioned(partition_kind, 2); + let second_tick = first_tick + .checked_add(1) + .expect("the fixture Tick can advance"); + let second_batch = inbox.admit_partitioned(partition_kind, 2, second_tick); assert_eq!(second_batch.len(), 2); assert!(second_batch.iter().all(|envelope| { matches!( @@ -1007,6 +1029,51 @@ mod tests { assert!(inbox.is_empty()); } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn partitioned_admission_cannot_starve_other_category_by_ingress_hash() { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::AcceptAll, + ); + let partition_kind = test_kind(); + let legacy = (0..256_u16) + .map(|index| make_envelope(other_kind(), format!("legacy-{index}").as_bytes())) + .max_by_key(IngressEnvelope::ingress_id) + .expect("the deterministic legacy candidate set is nonempty"); + let legacy_ingress_id = legacy.ingress_id(); + assert_eq!(inbox.ingest(legacy), InboxIngestResult::Accepted); + + let mut legacy_admitted = false; + for round in 0..2_u16 { + let partition = (0..1024_u16) + .map(|index| { + make_envelope( + partition_kind, + format!("partition-{round}-{index}").as_bytes(), + ) + }) + .find(|envelope| envelope.ingress_id() < legacy_ingress_id) + .expect("the deterministic partition candidates include a lower ingress hash"); + assert_eq!(inbox.ingest(partition), InboxIngestResult::Accepted); + let batch = inbox.admit_partitioned( + partition_kind, + 1, + crate::WorldlineTick::from_raw(u64::from(round)), + ); + assert_eq!(batch.len(), 1); + legacy_admitted |= batch[0].ingress_id() == legacy_ingress_id; + } + + assert!( + legacy_admitted, + "a continuously replenished lower-hash partition must not starve legacy work" + ); + } + #[test] fn re_ingesting_same_envelope_is_idempotent() { let mut inbox = HeadInbox::new( diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md index 6ecc79b1..4de30748 100644 --- a/docs/adr/0025-scheduler-owned-executable-operation-actions.md +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -85,11 +85,14 @@ Provider/native ingress and executable-operation Actions are different execution categories. A head inbox may contain both while migrations are in flight, but one Tick candidate batch is homogeneous. -The scheduler examines the lowest canonical ingress identity and drains only -that entry's execution category, subject to the existing inbox budget. Other -categories remain pending for a later Tick. This is deterministic, preserves -global inbox order at category boundaries, and makes it impossible for an -executable Action to fall through to a native callback engine. +When both categories are pending, the exact parent worldline-Tick parity selects +the category: an even parent Tick selects executable Actions and an odd parent +Tick selects provider/native ingress. When only one category is pending, that +category proceeds immediately. Entries remain canonically ordered by ingress +identity inside the selected category and existing inbox budgets still apply. +This durable alternating choice prevents caller-controlled hashes from starving +one evaluator category, remains stable across restart, and makes it impossible +for an executable Action to fall through to a native callback engine. ### One exact parent basis, per-Action application propositions diff --git a/docs/spec/scheduler-warp-core.md b/docs/spec/scheduler-warp-core.md index 9eb7c8ce..26e6c4c2 100644 --- a/docs/spec/scheduler-warp-core.md +++ b/docs/spec/scheduler-warp-core.md @@ -68,12 +68,13 @@ package and admission policy, but neither submission nor admission evaluates application semantics. The scheduler partitions executable and legacy/native ingress deterministically -at the lowest canonical ingress id so the two evaluator categories never share -one candidate batch. Inside an executable batch, Action ingress order is -canonical. Private bounded evaluation yields either a complete prepared -candidate or a typed no-mutation obstruction. The scheduler reserves successful -candidate footprints, constructs one composite consequence, and emits one Tick -receipt entry per Action. +so the two evaluator categories never share one candidate batch. When both are +pending, exact parent worldline-Tick parity alternates the selected category; +when only one is pending, it proceeds immediately. Inside either category, +ingress order is canonical. Private bounded evaluation yields either a complete +prepared candidate or a typed no-mutation obstruction. The scheduler reserves +successful candidate footprints, constructs one composite consequence, and +emits one Tick receipt entry per Action. The successor state remains private until the complete Tick transaction is durable. Construction failure discards it. WAL failure restores the From 305600ddf2a4e05e08b21c615c85a7363892e756 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 19:48:32 -0700 Subject: [PATCH 40/64] Fix: restrict positional Action attribution --- CHANGELOG.md | 2 + crates/warp-core/src/coordinator.rs | 59 +++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee2c3d85..d8b5041c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ Mixed executable and provider/native backlogs alternate by durable parent worldline-Tick parity, preventing caller-controlled ingress hashes from starving either execution category. + Positional receipt attribution is exclusive to executable Actions; ordinary + correlations without an exact scope match fail closed. Scheduler selection admits at most 64 executable Actions per Tick, leaving excess work pending, and meters footprint comparisons, blocker evidence, and aggregate operations during composition. Two independent Actions can share diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index e5017706..2c55bcd0 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -1825,10 +1825,7 @@ impl WorldlineRuntime { } candidate } else { - let Some(candidate) = positional_candidate() else { - return no_match(); - }; - candidate + return no_match(); }; let (idx, entry) = candidate; let Ok(receipt_entry_index) = u32::try_from(idx) else { @@ -6638,6 +6635,60 @@ mod tests { assert!(obstruction.contract.is_some()); } + #[test] + fn ordinary_outcome_never_uses_positional_action_attribution() { + let (mut runtime, mut engine, worldline_id) = toy_contract_runtime(); + let head_key = runtime + .resolve_target(&IngressTarget::DefaultWriter { worldline_id }) + .expect("the toy runtime has a default writer"); + let envelope = IngressEnvelope::local_intent( + IngressTarget::DefaultWriter { worldline_id }, + make_intent_kind("echo.intent/eint-v1"), + toy_increment_intent(TOY_INCREMENT_VARS), + ); + let mut provenance = mirrored_provenance(&runtime); + let mut correlation = commit_ticketed_envelope( + &mut runtime, + &mut provenance, + &mut engine, + head_key, + envelope, + hash(91), + ); + let receipt = &runtime + .worldlines() + .get(&worldline_id) + .expect("the toy worldline remains registered") + .state() + .tick_history() + .last() + .expect("the toy intent commits one Tick") + .1; + assert_eq!(receipt.entries().len(), 1); + correlation.ingress_id = (92_u8..=u8::MAX) + .map(hash) + .find(|candidate| { + receipt + .entries() + .iter() + .all(|entry| entry.scope.local_id != NodeId(*candidate)) + }) + .expect("the fixture provides an ingress id outside the receipt scopes"); + runtime + .receipt_correlations_by_ticketed_ingress + .insert(correlation.ticketed_ingress_id, correlation.clone()); + + assert!(matches!( + runtime.observe_intent_outcome(&correlation.submission_id), + IntentOutcomeObservation::Decided { + decision: IntentOutcomeDecision::NoMatchingReceiptEntry { + tick_receipt_digest, + }, + .. + } if tick_receipt_digest == correlation.tick_receipt_digest + )); + } + #[test] fn submission_order_does_not_define_scheduler_order() { let worldline_id = wl(1); From 9f5ce098be66d762729d7a70e8bb98c2e7b0dedd Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 19:55:34 -0700 Subject: [PATCH 41/64] Fix: index scheduler Action persistence --- CHANGELOG.md | 2 + crates/warp-core/src/coordinator.rs | 55 +++++++-- crates/warp-core/src/trusted_runtime_host.rs | 105 +++++++++--------- .../executable_operation_pipeline_tests.rs | 48 ++++++++ 4 files changed, 151 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8b5041c..50006ee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ constructing a Tick (ADR 0025). Accepted pre-Tick Actions recover as pending work. Runtime-owned admission uses a bounded pending index and cache; unavailable packages are quarantined without poisoning unrelated work. + Durable-acceptance lookup and newly committed receipt correlations are + indexed directly, so scheduler Ticks do not replay or diff retained history. A WAL-enabled app surface rejects executable Actions submitted outside the durable acknowledgement boundary before witnessed intake can mutate. Mixed executable and provider/native backlogs alternate by durable parent diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 2c55bcd0..b2ae43c9 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -1177,6 +1177,8 @@ pub struct WorldlineRuntime { /// admitted by that commit. receipt_correlations_by_current_basis: BTreeMap<(WorldlineId, WorldlineTick, Hash), BTreeSet>, + #[cfg(any(test, feature = "host_test"))] + receipt_correlation_full_scan_count: std::cell::Cell, /// Scheduler fault evidence keyed by content-addressed fault id. scheduler_faults: BTreeMap, /// Active scoped scheduler faults by writer head. @@ -1583,9 +1585,29 @@ impl WorldlineRuntime { /// Iterates receipt correlations in deterministic ticketed-ingress id order. pub fn receipt_correlations(&self) -> impl Iterator { + #[cfg(any(test, feature = "host_test"))] + self.receipt_correlation_full_scan_count.set( + self.receipt_correlation_full_scan_count + .get() + .saturating_add(1), + ); self.receipt_correlations_by_ticketed_ingress.values() } + /// Resets test instrumentation for full receipt-correlation scans. + #[cfg(any(test, feature = "host_test"))] + pub fn reset_receipt_correlation_full_scan_count_for_test(&self) { + self.receipt_correlation_full_scan_count.set(0); + } + + /// Returns the number of full receipt-correlation scans since the last + /// instrumentation reset. + #[cfg(any(test, feature = "host_test"))] + #[must_use] + pub fn receipt_correlation_full_scan_count_for_test(&self) -> usize { + self.receipt_correlation_full_scan_count.get() + } + /// Returns the number of scheduler-owned receipt correlations. #[must_use] pub fn receipt_correlation_count(&self) -> usize { @@ -3010,7 +3032,8 @@ impl WorldlineRuntime { admitted: &[IngressEnvelope], context: ReceiptCorrelationCommitContext, rollback: &mut ReceiptCorrelationRollback, - ) -> Result<(), RuntimeError> { + ) -> Result, RuntimeError> { + let mut committed = Vec::new(); for envelope in admitted { let ingress_id = envelope.ingress_id(); let Some(ticketed_ingress_id) = self @@ -3105,6 +3128,7 @@ impl WorldlineRuntime { .pending_witnessed_submission_ids .contains(&ticketed_ingress.submission_id), }); + committed.push(record.clone()); self.receipt_correlations_by_ticketed_ingress .insert(ticketed_ingress_id, record); self.receipt_correlation_by_submission @@ -3120,7 +3144,7 @@ impl WorldlineRuntime { self.pending_witnessed_submission_ids .remove(&ticketed_ingress.submission_id); } - Ok(()) + Ok(committed) } fn resolve_target(&self, target: &IngressTarget) -> Result { @@ -4145,7 +4169,7 @@ impl SchedulerCoordinator { engine: &mut Engine, ) -> Result, RuntimeError> { Self::super_tick_inner(runtime, provenance, engine, None) - .map(|(records, _operation_outcomes)| records) + .map(|(records, _operation_outcomes, _correlations)| records) } /// Executes one scheduler pass with runtime-admitted executable-operation @@ -4157,7 +4181,14 @@ impl SchedulerCoordinator { engine: &mut Engine, operation_actions: &BTreeMap, evaluation_authority: &EchoOperationEvaluationAuthorityV1, - ) -> Result<(Vec, SchedulerOperationOutcomesV1), RuntimeError> { + ) -> Result< + ( + Vec, + SchedulerOperationOutcomesV1, + Vec, + ), + RuntimeError, + > { Self::super_tick_inner( runtime, provenance, @@ -4177,7 +4208,14 @@ impl SchedulerCoordinator { )>, #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] _operation_actions: Option<()>, - ) -> Result<(Vec, SchedulerOperationOutcomesV1), RuntimeError> { + ) -> Result< + ( + Vec, + SchedulerOperationOutcomesV1, + Vec, + ), + RuntimeError, + > { if let Some(fault_id) = runtime.runtime_fault { return Err(RuntimeError::SchedulerRuntimeFaultActive(fault_id)); } @@ -4188,6 +4226,7 @@ impl SchedulerCoordinator { let mut operation_outcomes = Vec::new(); #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] let operation_outcomes = Vec::new(); + let mut committed_correlations = Vec::new(); let keys: Vec = runtime.runnable.iter().copied().collect(); let next_global_tick = if let Some(next) = runtime.global_tick.checked_increment() { next @@ -4416,7 +4455,7 @@ impl SchedulerCoordinator { .ok_or(RuntimeError::FrontierTickOverflow(key.worldline_id))?; (snapshot.state_root, worldline_tick_after) }; - runtime.record_receipt_correlations( + committed_correlations.extend(runtime.record_receipt_correlations( &admitted, ReceiptCorrelationCommitContext { head_key: *key, @@ -4426,7 +4465,7 @@ impl SchedulerCoordinator { commit_hash: snapshot.hash, }, &mut receipt_correlation_rollback, - )?; + )?); Ok(StepRecord { head_key: *key, @@ -4470,7 +4509,7 @@ impl SchedulerCoordinator { } runtime.global_tick = next_global_tick; - Ok((records, operation_outcomes)) + Ok((records, operation_outcomes, committed_correlations)) } /// Returns the canonical ordering of runnable heads without mutating state. diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 61994169..697c73a1 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -32,13 +32,12 @@ use crate::{ AffectedFrontierKind, FilesystemWalStore, InMemoryWalStore, Lsn, PayloadCodecId, PayloadSchemaId, RecoveredCausalAnchorAdmission, RecoveredReceiptIndex, RecoveredSubmissionIndex, RecoveryAccessMode, RecoveryCertificate, RecoveryScanReport, - SubmissionAcceptanceRecord, SubmissionRetryPosture, TickReceiptRecord, WalAppendAuthority, - WalBuildError, WalCommittedTransaction, WalDecodeError, WalDurabilityMode, - WalReceiptCorrelationRecord, WalRecordKind, WalRecoveryError, WalRecoveryIndexError, - WalRuntimeStateDeltaRecord, WalSegmentId, WalStoreError, WalStorePort, - WalSubmissionEnvelopeRecord, WalTickDecision, WalTransactionBuilder, WalTransactionCommit, - WalTransactionId, WalTransactionKind, WriterEpochId, WriterEpochRequest, - TRUSTED_RUNTIME_WAL_DOMAIN, + SubmissionAcceptanceRecord, TickReceiptRecord, WalAppendAuthority, WalBuildError, + WalCommittedTransaction, WalDecodeError, WalDurabilityMode, WalReceiptCorrelationRecord, + WalRecordKind, WalRecoveryError, WalRecoveryIndexError, WalRuntimeStateDeltaRecord, + WalSegmentId, WalStoreError, WalStorePort, WalSubmissionEnvelopeRecord, WalTickDecision, + WalTransactionBuilder, WalTransactionCommit, WalTransactionId, WalTransactionKind, + WriterEpochId, WriterEpochRequest, TRUSTED_RUNTIME_WAL_DOMAIN, }, contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ @@ -1774,7 +1773,7 @@ impl TrustedRuntimeHost { .to_vec(); if let Some(runtime_wal) = self.runtime_wal.as_ref() { let durably_accepted = runtime_wal - .has_submission_acceptance(submission.submission_id, submission.ingress_id)?; + .has_submission_acceptance(submission.submission_id, submission.ingress_id); if !durably_accepted { continue; } @@ -1838,12 +1837,7 @@ impl TrustedRuntimeHost { return Err(error); } }; - let existing_correlations = self - .runtime - .receipt_correlations() - .map(|correlation| correlation.ticketed_ingress_id) - .collect::>(); - let (records, action_outcomes) = + let (records, action_outcomes, new_correlations) = SchedulerCoordinator::super_tick_with_echo_operation_actions_v1( &mut self.runtime, &mut self.provenance, @@ -1851,12 +1845,6 @@ impl TrustedRuntimeHost { &admitted_actions, &self.echo_operation_evaluation_authority, )?; - let new_correlations = self - .runtime - .receipt_correlations() - .filter(|correlation| !existing_correlations.contains(&correlation.ticketed_ingress_id)) - .cloned() - .collect::>(); let mut tick_wal_records = Vec::new(); if self.runtime_wal.is_some() { for correlation in new_correlations { @@ -2239,7 +2227,7 @@ pub struct TrustedRuntimeWal { evidence_catalog_posture: EvidenceCatalogPosture, #[cfg(any(test, feature = "host_test"))] fail_next_evidence_catalog_update: bool, - #[cfg(test)] + #[cfg(any(test, feature = "host_test"))] recover_read_only_call_count: std::cell::Cell, writer_epoch: WriterEpochId, segment_id: WalSegmentId, @@ -2258,6 +2246,7 @@ pub struct TrustedRuntimeWal { causal_anchor_frontier_digest: Hash, causal_history_frontier_digest: Hash, causal_anchor_claim_projection: CausalAnchorClaimProjection, + durable_submission_acceptances: BTreeMap, } impl TrustedRuntimeWal { @@ -2277,6 +2266,13 @@ impl TrustedRuntimeWal { let mut store = TrustedRuntimeWalStore::open(store)?; let recovery_report = store.recover_for_writer()?; let recovered_cursor = TrustedRuntimeWalCursor::from_recovery(&recovery_report)?; + let durable_submission_acceptances = recover_submission_index(&recovery_report) + .map_err(WalRecoveryError::from)? + .entries() + .map(|(submission_id, entry)| { + (*submission_id, entry.acceptance.canonical_envelope_digest) + }) + .collect(); let evidence_catalog = crate::evidence::CausalSegmentCatalog::from_recovery_scan(&recovery_report)?; let next_lsn = if recovered_cursor.has_committed_history { @@ -2322,11 +2318,12 @@ impl TrustedRuntimeWal { causal_anchor_frontier_digest: recovered_cursor.causal_anchor_frontier_digest, causal_history_frontier_digest: recovered_cursor.causal_history_frontier_digest, causal_anchor_claim_projection: recovered_cursor.causal_anchor_claim_projection, + durable_submission_acceptances, evidence_catalog: Some(evidence_catalog), evidence_catalog_posture: EvidenceCatalogPosture::Fresh, #[cfg(any(test, feature = "host_test"))] fail_next_evidence_catalog_update: false, - #[cfg(test)] + #[cfg(any(test, feature = "host_test"))] recover_read_only_call_count: std::cell::Cell::new(0), }) } @@ -2379,7 +2376,7 @@ impl TrustedRuntimeWal { /// Recovers submission and receipt indexes from committed WAL transactions /// without scheduler callbacks. pub fn recover_read_only(&self) -> Result { - #[cfg(test)] + #[cfg(any(test, feature = "host_test"))] self.recover_read_only_call_count .set(self.recover_read_only_call_count.get() + 1); let report = self.store.recover_read_only()?; @@ -2445,6 +2442,20 @@ impl TrustedRuntimeWal { }) } + /// Resets test instrumentation for full read-only WAL recovery. + #[cfg(any(test, feature = "host_test"))] + pub fn reset_recover_read_only_call_count_for_test(&self) { + self.recover_read_only_call_count.set(0); + } + + /// Returns the number of full read-only WAL recoveries since the last + /// instrumentation reset. + #[cfg(any(test, feature = "host_test"))] + #[must_use] + pub fn recover_read_only_call_count_for_test(&self) -> usize { + self.recover_read_only_call_count.get() + } + /// Recovers the causal segment catalog from committed WAL transactions. pub fn recover_evidence_catalog_read_only( &self, @@ -2524,17 +2535,8 @@ impl TrustedRuntimeWal { &self, submission_id: Hash, canonical_envelope_digest: Hash, - ) -> Result { - let recovery = self.recover_read_only()?; - Ok(matches!( - recovery - .submissions - .retry_posture(submission_id, canonical_envelope_digest), - SubmissionRetryPosture::AlreadyAcceptedPending - | SubmissionRetryPosture::AlreadyDecidedApplied - | SubmissionRetryPosture::AlreadyDecidedRejected - | SubmissionRetryPosture::AlreadyObstructed - )) + ) -> bool { + self.durable_submission_acceptances.get(&submission_id) == Some(&canonical_envelope_digest) } fn uses_filesystem_store(&self) -> bool { @@ -2554,6 +2556,13 @@ impl TrustedRuntimeWal { fn refresh_cursor_from_store_for_writer(&mut self) -> Result<(), TrustedRuntimeWalError> { let report = self.store.recover_for_writer()?; let cursor = TrustedRuntimeWalCursor::from_recovery(&report)?; + let durable_submission_acceptances = recover_submission_index(&report) + .map_err(WalRecoveryError::from)? + .entries() + .map(|(submission_id, entry)| { + (*submission_id, entry.acceptance.canonical_envelope_digest) + }) + .collect(); match crate::evidence::CausalSegmentCatalog::from_recovery_scan(&report) { Ok(catalog) => { self.evidence_catalog = Some(catalog); @@ -2579,6 +2588,7 @@ impl TrustedRuntimeWal { self.causal_anchor_frontier_digest = cursor.causal_anchor_frontier_digest; self.causal_history_frontier_digest = cursor.causal_history_frontier_digest; self.causal_anchor_claim_projection = cursor.causal_anchor_claim_projection; + self.durable_submission_acceptances = durable_submission_acceptances; Ok(()) } @@ -2594,7 +2604,6 @@ impl TrustedRuntimeWal { return false; } self.has_submission_acceptance(submission_id, canonical_envelope_digest) - .unwrap_or(false) } fn recover_filesystem_tick_commit_after_error( @@ -2698,6 +2707,8 @@ impl TrustedRuntimeWal { )?; let commit = self.append_transaction(transaction)?; self.submission_frontier_digest = next_submission_frontier; + self.durable_submission_acceptances + .insert(record.submission_id, record.canonical_envelope_digest); Ok(commit) } @@ -5560,22 +5571,14 @@ impl TrustedRuntimeApp<'_> { self.host.runtime = before_runtime; return Err(TrustedRuntimeHostError::RuntimeWalUnavailable); }; - if handle.duplicate { - match runtime_wal.has_submission_acceptance(handle.submission_id, envelope.ingress_id()) - { - Ok(true) => { - self.host.track_pending_echo_operation_action_v1( - handle.submission_id, - is_echo_operation_action, - ); - return Ok(handle); - } - Ok(false) => {} - Err(error) => { - self.host.runtime = before_runtime; - return Err(error.into()); - } - } + if handle.duplicate + && runtime_wal.has_submission_acceptance(handle.submission_id, envelope.ingress_id()) + { + self.host.track_pending_echo_operation_action_v1( + handle.submission_id, + is_echo_operation_action, + ); + return Ok(handle); } if let Err(error) = runtime_wal.record_submission_acceptance(&envelope, handle) { if runtime_wal.recover_filesystem_submission_acceptance_after_error( diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index e2079c20..a43d2897 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3236,6 +3236,54 @@ fn wal_enabled_host_rejects_action_submission_without_durable_ack() { assert!(recovery.echo_operation_action_outcomes.is_empty()); } +#[cfg(feature = "host_test")] +#[test] +fn scheduler_action_hot_path_uses_bounded_indexes() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the indexed scheduler fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"indexed-after", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes an Action"); + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the Action acceptance is durable"); + host.runtime_wal() + .expect("the runtime WAL remains enabled") + .reset_recover_read_only_call_count_for_test(); + host.runtime() + .reset_receipt_correlation_full_scan_count_for_test(); + + host.tick_once() + .expect("the indexed Action reaches one scheduler Tick"); + + assert_eq!( + host.runtime_wal() + .expect("the runtime WAL remains enabled") + .recover_read_only_call_count_for_test(), + 0, + "Action admission must not replay all committed WAL history" + ); + assert_eq!( + host.runtime() + .receipt_correlation_full_scan_count_for_test(), + 0, + "Tick persistence must consume newly committed correlations directly" + ); +} + #[test] fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { let wal_dir = TempWalDir::new(); From f5c8cd355cf9f1a83263bb79239a70ba16c8630d Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:04:46 -0700 Subject: [PATCH 42/64] Fix: index Action installation order --- CHANGELOG.md | 3 + crates/warp-core/src/trusted_runtime_host.rs | 85 ++++++++++++++----- .../executable_operation_pipeline_tests.rs | 5 ++ 3 files changed, 71 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50006ee6..325fcc53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ Typed obstruction records retain their invocation-admission policy and budget ceiling; recovery reproduces bounded evaluation and the complete scheduler composition before accepting an obstruction kind. + Recovery records one monotonic installation ordinal per package and one + installation-count boundary per Action, avoiding per-outcome package-set + snapshots while preserving exact installation-before-Tick validation. A composite receipt cannot validate outside its complete Action-batch context. Legacy operation recovery-index roots remain byte-compatible when no Action outcome exists. Direct operation diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 697c73a1..798a2254 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -427,6 +427,12 @@ impl WitnessedCausalAnchorAdmission { } } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct RecoveredEchoOperationActionInstallationOrder { + package_ordinals: BTreeMap, + installation_count_before_tick: BTreeMap, +} + /// Read-only runtime WAL recovery report. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TrustedRuntimeWalRecovery { @@ -456,8 +462,9 @@ pub struct TrustedRuntimeWalRecovery { /// scheduler-owned Tick records, keyed by witnessed submission. pub echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, echo_operation_action_decisions: BTreeMap, - echo_operation_action_installations_before_tick: - BTreeMap>, + echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder, + #[cfg(any(test, feature = "host_test"))] + echo_operation_action_installation_snapshot_count: usize, causal_history_frontiers: Vec, } @@ -475,7 +482,7 @@ impl TrustedRuntimeWalRecovery { &self.installed_echo_operations, &self.echo_operation_action_outcomes, &self.echo_operation_action_decisions, - &self.echo_operation_action_installations_before_tick, + &self.echo_operation_action_installation_order, ) } @@ -496,10 +503,19 @@ impl TrustedRuntimeWalRecovery { &mut self, submission_id: Hash, ) { - self.echo_operation_action_installations_before_tick + self.echo_operation_action_installation_order + .installation_count_before_tick .remove(&submission_id); } + /// Returns the number of per-Action installation-set snapshots constructed + /// during recovery. + #[cfg(any(test, feature = "host_test"))] + #[must_use] + pub fn echo_operation_action_installation_snapshot_count_for_test(&self) -> usize { + self.echo_operation_action_installation_snapshot_count + } + /// Replaces one retained obstruction kind for adversarial recovery tests. #[cfg(any(test, feature = "host_test"))] pub fn replace_echo_operation_action_obstruction_kind_for_test( @@ -2393,8 +2409,11 @@ impl TrustedRuntimeWal { let receipt_correlations = runtime_state.receipt_correlations; let echo_operation_action_outcomes = runtime_state.echo_operation_action_outcomes; let echo_operation_action_decisions = runtime_state.echo_operation_action_decisions; - let echo_operation_action_installations_before_tick = - runtime_state.echo_operation_action_installations_before_tick; + let echo_operation_action_installation_order = + runtime_state.echo_operation_action_installation_order; + #[cfg(any(test, feature = "host_test"))] + let echo_operation_action_installation_snapshot_count = + runtime_state.echo_operation_action_installation_snapshot_count; let missing_runtime_state_deltas = runtime_state.missing_runtime_state_deltas; let installed_echo_operations = operation_material.installations; let echo_operation_receipts = operation_material.receipts; @@ -2406,7 +2425,7 @@ impl TrustedRuntimeWal { &installed_echo_operations, &echo_operation_action_outcomes, &echo_operation_action_decisions, - &echo_operation_action_installations_before_tick, + &echo_operation_action_installation_order, )?; let certificate = runtime_wal_recovery_certificate( &report, @@ -2437,7 +2456,9 @@ impl TrustedRuntimeWal { echo_operation_receipts, echo_operation_action_outcomes, echo_operation_action_decisions, - echo_operation_action_installations_before_tick, + echo_operation_action_installation_order, + #[cfg(any(test, feature = "host_test"))] + echo_operation_action_installation_snapshot_count, causal_history_frontiers, }) } @@ -4546,8 +4567,9 @@ struct RecoveredRuntimeStateMaterial { receipt_correlations: Vec, echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, echo_operation_action_decisions: BTreeMap, - echo_operation_action_installations_before_tick: - BTreeMap>, + echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder, + #[cfg(any(test, feature = "host_test"))] + echo_operation_action_installation_snapshot_count: usize, missing_runtime_state_deltas: Vec, } @@ -4558,8 +4580,10 @@ fn recover_runtime_state_delta_material( let mut correlations_by_submission = BTreeMap::new(); let mut action_outcomes_by_submission = BTreeMap::new(); let mut action_decisions_by_submission = BTreeMap::new(); - let mut action_installations_before_tick = BTreeMap::new(); - let mut installed_packages = BTreeSet::new(); + let mut action_installation_count_before_tick = BTreeMap::new(); + #[cfg(any(test, feature = "host_test"))] + let action_installation_snapshot_count = 0_usize; + let mut installation_ordinals = BTreeMap::new(); let mut submission_by_ticket = BTreeMap::new(); let mut missing = Vec::new(); for transaction in &report.transactions { @@ -4567,7 +4591,10 @@ fn recover_runtime_state_delta_material( == WalTransactionKind::ExecutableOperationInstallation { let (installed, _) = operation_installation_from_transaction(transaction)?; - installed_packages.insert(installed.package_id()); + let next_ordinal = installation_ordinals.len(); + installation_ordinals + .entry(installed.package_id()) + .or_insert(next_ordinal); continue; } if transaction.commit.transaction_kind == WalTransactionKind::ExecutableOperationTick { @@ -4669,18 +4696,21 @@ fn recover_runtime_state_delta_material( } correlations_by_submission.insert(receipt.receipt_ref.submission_id, persistence); if let Some((submission_id, ingress_id, outcome)) = action_outcome { + let decision_conflicts = action_decisions_by_submission + .get(&submission_id) + .is_some_and(|existing| *existing != receipt.decision); if submission_id != receipt.receipt_ref.submission_id || action_outcomes_by_submission .get(&submission_id) .is_some_and(|existing| existing != &(ingress_id, outcome.clone())) - || action_decisions_by_submission - .insert(submission_id, receipt.decision) - .is_some_and(|existing| existing != receipt.decision) + || decision_conflicts { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } + action_decisions_by_submission.insert(submission_id, receipt.decision); action_outcomes_by_submission.insert(submission_id, (ingress_id, outcome)); - action_installations_before_tick.insert(submission_id, installed_packages.clone()); + action_installation_count_before_tick + .insert(submission_id, installation_ordinals.len()); } } entries_by_coordinate.insert(coordinate, entry); @@ -4713,7 +4743,12 @@ fn recover_runtime_state_delta_material( receipt_correlations: correlations, echo_operation_action_outcomes, echo_operation_action_decisions: action_decisions_by_submission, - echo_operation_action_installations_before_tick: action_installations_before_tick, + echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder { + package_ordinals: installation_ordinals, + installation_count_before_tick: action_installation_count_before_tick, + }, + #[cfg(any(test, feature = "host_test"))] + echo_operation_action_installation_snapshot_count: action_installation_snapshot_count, missing_runtime_state_deltas: missing, }) } @@ -4750,7 +4785,7 @@ fn validate_recovered_echo_operation_action_outcomes( installed_echo_operations: &[InstalledEchoOperationV1], outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], decisions: &BTreeMap, - installations_before_tick: &BTreeMap>, + installation_order: &RecoveredEchoOperationActionInstallationOrder, ) -> Result<(), TrustedRuntimeWalError> { let envelopes = witnessed_submissions .records() @@ -4815,10 +4850,16 @@ fn validate_recovered_echo_operation_action_outcomes( .iter() .find(|installed| installed.package_id() == invocation.package_id) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; - if installations_before_tick + let installed_before_tick = installation_order + .installation_count_before_tick .get(submission_id) - .is_none_or(|packages| !packages.contains(&invocation.package_id)) - { + .zip( + installation_order + .package_ordinals + .get(&invocation.package_id), + ) + .is_some_and(|(count, ordinal)| ordinal < count); + if !installed_before_tick { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } if envelope.ingress_id() != *ingress_id { diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index a43d2897..ff943872 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2909,6 +2909,11 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { let mut adversarial = runtime_wal .recover_read_only() .expect("the honest composite Tick recovers"); + assert_eq!( + adversarial.echo_operation_action_installation_snapshot_count_for_test(), + 0, + "recovery must not clone the installed-package set per Action" + ); let mut missing_installation = adversarial.clone(); missing_installation.installed_echo_operations.clear(); assert!(matches!( From 7f208f35ce3e9a9d504f994a0b95dcd9098305d0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:09:12 -0700 Subject: [PATCH 43/64] Fix: cache recovered Action basis states --- CHANGELOG.md | 2 + crates/warp-core/src/trusted_runtime_host.rs | 90 ++++++++++++++----- .../executable_operation_pipeline_tests.rs | 10 +++ 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 325fcc53..da8e7fd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ Recovery records one monotonic installation ordinal per package and one installation-count boundary per Action, avoiding per-outcome package-set snapshots while preserving exact installation-before-Tick validation. + Activation caches each reconstructed causal state by worldline coordinate, + so Actions sharing one scheduler basis do not replay that history repeatedly. A composite receipt cannot validate outside its complete Action-batch context. Legacy operation recovery-index roots remain byte-compatible when no Action outcome exists. Direct operation diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 798a2254..a24942d9 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -571,6 +571,17 @@ impl TrustedRuntimeWalRecovery { runtime: &WorldlineRuntime, provenance: &ProvenanceService, ) -> Result<(), TrustedRuntimeWalError> { + validate_recovered_echo_operation_parent_states(runtime, provenance, self).map(|_| ()) + } + + /// Returns the number of causal-state reconstructions used by Action + /// parent-state validation. + #[cfg(any(test, feature = "host_test"))] + pub fn echo_operation_parent_state_replay_count_for_test( + &self, + runtime: &WorldlineRuntime, + provenance: &ProvenanceService, + ) -> Result { validate_recovered_echo_operation_parent_states(runtime, provenance, self) } @@ -3931,11 +3942,36 @@ fn operation_application_basis_matches_scope_v1( } } +fn recovered_worldline_state_at<'a>( + cache: &'a mut BTreeMap<(crate::WorldlineId, crate::WorldlineTick), crate::WorldlineState>, + recovered_provenance: &ProvenanceService, + worldline_id: crate::WorldlineId, + frontier_state: &crate::WorldlineState, + worldline_tick: crate::WorldlineTick, + failure_detail: &'static str, +) -> Result<&'a crate::WorldlineState, TrustedRuntimeWalError> { + let coordinate = (worldline_id, worldline_tick); + if let std::collections::btree_map::Entry::Vacant(entry) = cache.entry(coordinate) { + let state = recovered_provenance + .replay_worldline_state_at(worldline_id, frontier_state, worldline_tick) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: failure_detail, + })?; + entry.insert(state); + } + cache + .get(&coordinate) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: failure_detail, + }) +} + fn validate_recovered_echo_operation_parent_states( runtime: &WorldlineRuntime, recovered_provenance: &ProvenanceService, recovery: &TrustedRuntimeWalRecovery, -) -> Result<(), TrustedRuntimeWalError> { +) -> Result { + let mut recovered_states = BTreeMap::new(); let installations = recovery .installed_echo_operations .iter() @@ -3955,11 +3991,14 @@ fn validate_recovered_echo_operation_parent_states( detail: "operation receipt names an unavailable recovery worldline", }, )?; - let parent_state = recovered_provenance - .replay_worldline_state_at(worldline_id, frontier.state(), basis.worldline_tick()) - .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "operation receipt parent state cannot be reconstructed", - })?; + let parent_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + worldline_id, + frontier.state(), + basis.worldline_tick(), + "operation receipt parent state cannot be reconstructed", + )?; let entry = entries.get(&(worldline_id, basis.worldline_tick())).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "operation receipt has no recovered state transition", @@ -3980,7 +4019,7 @@ fn validate_recovered_echo_operation_parent_states( patch, receipt.installed_operation_id().as_hash(), installed.program(), - &parent_state, + parent_state, ); let application_basis_matches = exact_scope.is_some_and(|node| { operation_application_basis_matches_scope_v1( @@ -4059,13 +4098,16 @@ fn validate_recovered_echo_operation_parent_states( detail: "Action basis names an unavailable recovery worldline", }, )?; - let basis_state = recovered_provenance - .replay_worldline_state_at(worldline_id, frontier.state(), basis.worldline_tick()) - .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action basis state cannot be reconstructed", - })?; + let basis_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + worldline_id, + frontier.state(), + basis.worldline_tick(), + "Action basis state cannot be reconstructed", + )?; if basis_state.state_root() != basis.state_root() - || !action_application_basis_matches_state_v1(installed, invocation_bytes, &basis_state) + || !action_application_basis_matches_state_v1(installed, invocation_bytes, basis_state) .unwrap_or(false) || !evaluation_basis_matches_recovered_coordinate(basis, &entries) { @@ -4113,7 +4155,7 @@ fn validate_recovered_echo_operation_parent_states( receipt.retained_invocation_admission_maximum_budget(), receipt.retained_invocation_admission_policy_id(), receipt.invocation_admission_id(), - &basis_state, + basis_state, policy_id, ) .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { @@ -4147,7 +4189,7 @@ fn validate_recovered_echo_operation_parent_states( conflict.invocation_admission_maximum_budget, conflict.invocation_admission_policy_id, conflict.invocation_admission_id, - &basis_state, + basis_state, policy_id, ) .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { @@ -4198,7 +4240,7 @@ fn validate_recovered_echo_operation_parent_states( obstruction.invocation_admission_maximum_budget(), obstruction.invocation_admission_policy_id(), obstruction.invocation_admission_id(), - &basis_state, + basis_state, policy_id, ) .ok_or( @@ -4269,11 +4311,15 @@ fn validate_recovered_echo_operation_parent_states( detail: "Action Tick names an unavailable recovery worldline", }, )?; - let mut reconstructed_state = recovered_provenance - .replay_worldline_state_at(head_key.worldline_id, frontier.state(), tick_before) - .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action Tick parent state cannot be reconstructed", - })?; + let mut reconstructed_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + head_key.worldline_id, + frontier.state(), + tick_before, + "Action Tick parent state cannot be reconstructed", + )? + .clone(); let candidates = members .iter() .map( @@ -4417,7 +4463,7 @@ fn validate_recovered_echo_operation_parent_states( } } } - Ok(()) + Ok(recovered_states.len()) } fn evaluation_basis_matches_recovered_coordinate( diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index ff943872..2949a21e 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2914,6 +2914,16 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { 0, "recovery must not clone the installed-package set per Action" ); + assert_eq!( + adversarial + .echo_operation_parent_state_replay_count_for_test( + host.runtime(), + host.provenance() + ) + .expect("the honest composite Tick parent states validate"), + 1, + "one shared basis must be reconstructed once for the whole Action batch" + ); let mut missing_installation = adversarial.clone(); missing_installation.installed_echo_operations.clear(); assert!(matches!( From dd774096e40e34b3df6a8390198a0a6516489324 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:11:45 -0700 Subject: [PATCH 44/64] Fix: index recovered Action installations --- CHANGELOG.md | 2 + crates/warp-core/src/trusted_runtime_host.rs | 39 ++++++++++++++++--- .../executable_operation_pipeline_tests.rs | 7 ++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da8e7fd9..3f64dbaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ snapshots while preserving exact installation-before-Tick validation. Activation caches each reconstructed causal state by worldline coordinate, so Actions sharing one scheduler basis do not replay that history repeatedly. + Recovered Action outcomes resolve installed packages through one package-ID + index instead of scanning the complete installation set per Action. A composite receipt cannot validate outside its complete Action-batch context. Legacy operation recovery-index roots remain byte-compatible when no Action outcome exists. Direct operation diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index a24942d9..9987d008 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -433,6 +433,12 @@ struct RecoveredEchoOperationActionInstallationOrder { installation_count_before_tick: BTreeMap, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct RecoveredEchoOperationActionOutcomeValidationMetrics { + #[cfg(any(test, feature = "host_test"))] + linear_installation_comparisons: usize, +} + /// Read-only runtime WAL recovery report. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TrustedRuntimeWalRecovery { @@ -484,6 +490,25 @@ impl TrustedRuntimeWalRecovery { &self.echo_operation_action_decisions, &self.echo_operation_action_installation_order, ) + .map(|_| ()) + } + + /// Returns the number of linear installed-package comparisons used while + /// validating recovered Action outcomes. + #[cfg(any(test, feature = "host_test"))] + pub fn echo_operation_action_linear_installation_comparisons_for_test( + &self, + ) -> Result { + validate_recovered_echo_operation_action_outcomes( + &self.witnessed_submissions, + &self.receipt_correlations, + &self.provenance_entries, + &self.installed_echo_operations, + &self.echo_operation_action_outcomes, + &self.echo_operation_action_decisions, + &self.echo_operation_action_installation_order, + ) + .map(|metrics| metrics.linear_installation_comparisons) } /// Replaces one recovered scheduler decision for adversarial recovery tests. @@ -4832,7 +4857,8 @@ fn validate_recovered_echo_operation_action_outcomes( outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], decisions: &BTreeMap, installation_order: &RecoveredEchoOperationActionInstallationOrder, -) -> Result<(), TrustedRuntimeWalError> { +) -> Result { + let metrics = RecoveredEchoOperationActionOutcomeValidationMetrics::default(); let envelopes = witnessed_submissions .records() .iter() @@ -4861,6 +4887,10 @@ fn validate_recovered_echo_operation_action_outcomes( )) }) .collect::>(); + let installed_by_package = installed_echo_operations + .iter() + .map(|installed| (installed.package_id(), installed)) + .collect::>(); let mut outcomes_by_tick = BTreeMap::< ( crate::WriterHeadKey, @@ -4892,9 +4922,8 @@ fn validate_recovered_echo_operation_action_outcomes( .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; let invocation = inspect_action_invocation_v1(invocation_bytes) .map_err(|_| TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; - let installed = installed_echo_operations - .iter() - .find(|installed| installed.package_id() == invocation.package_id) + let installed = installed_by_package + .get(&invocation.package_id) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; let installed_before_tick = installation_order .installation_count_before_tick @@ -5091,7 +5120,7 @@ fn validate_recovered_echo_operation_action_outcomes( } } } - Ok(()) + Ok(metrics) } fn wal_tick_decision_for_action_outcome(outcome: &EchoOperationActionOutcomeV1) -> WalTickDecision { diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 2949a21e..403ae008 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2914,6 +2914,13 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { 0, "recovery must not clone the installed-package set per Action" ); + assert_eq!( + adversarial + .echo_operation_action_linear_installation_comparisons_for_test() + .expect("the honest composite Tick outcomes validate"), + 0, + "recovery must resolve installed packages through one identity index" + ); assert_eq!( adversarial .echo_operation_parent_state_replay_count_for_test( From e4db431120749ce9da35d93484e0bff81e422c8e Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:15:16 -0700 Subject: [PATCH 45/64] Fix: prove the scheduler Action candidate limit --- CHANGELOG.md | 3 + crates/warp-core/src/echo_operation.rs | 4 +- crates/warp-core/src/lib.rs | 2 +- .../executable_operation_pipeline_tests.rs | 69 +++++++++++++------ 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f64dbaa..cfe8558f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,9 @@ so Actions sharing one scheduler basis do not replay that history repeatedly. Recovered Action outcomes resolve installed packages through one package-ID index instead of scanning the complete installation set per Action. + The v1 scheduler Action-candidate ceiling is exported as + `ACTION_BATCH_CANDIDATE_LIMIT_V1`; its acceptance witness now proves the + complete limit with independent, non-conflicting node targets. A composite receipt cannot validate outside its complete Action-batch context. Legacy operation recovery-index roots remain byte-compatible when no Action outcome exists. Direct operation diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 280db04d..c0960baf 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -85,7 +85,9 @@ const ACTION_BATCH_PLAN_DOMAIN: &[u8] = b"echo:operation-action-batch-plan:v1\0" const ACTION_BATCH_REWRITES_DOMAIN: &[u8] = b"echo:operation-action-batch-rewrites:v1\0"; const ACTION_BATCH_COMPOSITION_DOMAIN: &[u8] = b"echo:operation-action-batch-composition:v1\0"; const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT003"; -pub(crate) const ACTION_BATCH_CANDIDATE_LIMIT_V1: usize = 64; +/// Maximum number of executable-operation Action candidates selected for one +/// scheduler-owned Tick under the v1 bounded-composition profile. +pub const ACTION_BATCH_CANDIDATE_LIMIT_V1: usize = 64; const ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1: usize = ACTION_BATCH_CANDIDATE_LIMIT_V1 * (ACTION_BATCH_CANDIDATE_LIMIT_V1 - 1) / 2; const ACTION_BATCH_BLOCKER_EVIDENCE_LIMIT_V1: usize = ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1; diff --git a/crates/warp-core/src/lib.rs b/crates/warp-core/src/lib.rs index 25156c50..c8406f29 100644 --- a/crates/warp-core/src/lib.rs +++ b/crates/warp-core/src/lib.rs @@ -258,7 +258,7 @@ pub use echo_operation::{ EchoOperationProgramV1, EchoOperationReceiptV1, EchoOperationResultIdV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, ExecutableOperationPackageV1, InstalledEchoOperationIdV1, InstalledEchoOperationV1, PreparedEchoOperationIdV1, - PreparedEchoOperationV1, + PreparedEchoOperationV1, ACTION_BATCH_CANDIDATE_LIMIT_V1, }; pub use edict_target_ir::{ accept_edict_echo_target_ir, execute_accepted_edict_echo_target_ir, AcceptedEdictEchoTargetIr, diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 403ae008..af806c5c 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -32,7 +32,7 @@ use warp_core::{ InstalledEchoOperationV1, NodeKey, NodeRecord, PlaybackMode, RuntimeError, RuntimeWalActivationGap, SchedulerKind, TrustedRuntimeHost, TrustedRuntimeHostError, TrustedRuntimeWalConfig, TrustedRuntimeWalError, WorldlineId, WorldlineRuntime, WorldlineState, - WriterHead, WriterHeadKey, + WriterHead, WriterHeadKey, ACTION_BATCH_CANDIDATE_LIMIT_V1, }; const OPERATION_COORDINATE: &str = "echo.fixture.SetAnchoredAtom.v1"; @@ -107,24 +107,44 @@ fn operation_frontier(kind: AffectedFrontierKind, label: &str) -> AffectedFronti } fn fixture_host() -> (TrustedRuntimeHost, WriterHeadKey, NodeKey) { + let (host, head_key, nodes) = fixture_host_with_independent_nodes(1); + ( + host, + head_key, + *nodes.first().expect("the fixture contains its root node"), + ) +} + +fn fixture_host_with_independent_nodes( + node_count: usize, +) -> (TrustedRuntimeHost, WriterHeadKey, Vec) { + assert!(node_count > 0, "the fixture must retain one root node"); let warp_id = make_warp_id("operation-fixture"); - let node_id = make_node_id("operation-fixture-root"); + let root_id = make_node_id("operation-fixture-root"); let node_type = make_type_id("operation-fixture-node"); let attachment_type = make_type_id("operation-fixture-atom"); - let node = NodeKey { - warp_id, - local_id: node_id, - }; + let nodes = (0..node_count) + .map(|index| NodeKey { + warp_id, + local_id: if index == 0 { + root_id + } else { + make_node_id(&format!("operation-fixture-node-{index}")) + }, + }) + .collect::>(); let mut store = GraphStore::new(warp_id); - store.insert_node(node_id, NodeRecord { ty: node_type }); - store.set_node_attachment( - node_id, - Some(AttachmentValue::Atom(AtomPayload::new( - attachment_type, - Bytes::from_static(b"before"), - ))), - ); - let state = WorldlineState::from_root_store(store, node_id) + for node in &nodes { + store.insert_node(node.local_id, NodeRecord { ty: node_type }); + store.set_node_attachment( + node.local_id, + Some(AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"before"), + ))), + ); + } + let state = WorldlineState::from_root_store(store, root_id) .expect("the fixture state has one lawful root"); let worldline_id = WorldlineId::from_bytes(digest("operation-fixture-worldline")); let head_key = WriterHeadKey { @@ -159,7 +179,7 @@ fn fixture_host() -> (TrustedRuntimeHost, WriterHeadKey, NodeKey) { .build(); let host = TrustedRuntimeHost::new(runtime, engine) .expect("the trusted Echo runtime host initializes"); - (host, head_key, node) + (host, head_key, nodes) } fn fixture_host_with_two_worldlines() -> (TrustedRuntimeHost, WriterHeadKey, WriterHeadKey, NodeKey) @@ -3066,14 +3086,15 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { #[test] fn scheduler_candidate_limit_leaves_excess_action_pending() { - let (mut host, head_key, node) = fixture_host(); + let candidate_count = ACTION_BATCH_CANDIDATE_LIMIT_V1 + 1; + let (mut host, head_key, nodes) = fixture_host_with_independent_nodes(candidate_count); host.enable_in_memory_runtime_wal() .expect("the scheduler budget fixture WAL opens"); let installed = install_fixture_operation(&mut host); host.install_echo_operation_action_admission_policy_v1(invocation_policy()); let mut submission_ids = Vec::new(); - for index in 0..65 { + for (index, node) in nodes.into_iter().enumerate() { let replacement = format!("bounded-after-{index}"); let invocation = action_invocation( &host, @@ -3095,13 +3116,19 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { .submission_id, ); } - assert_eq!(host.runtime().pending_witnessed_submission_count(), 65); + assert_eq!( + host.runtime().pending_witnessed_submission_count(), + candidate_count + ); let first_tick = host .tick_once() .expect("the first scheduler Tick respects its Action candidate limit"); assert_eq!(first_tick.len(), 1); - assert_eq!(first_tick[0].admitted_count, 64); + assert_eq!( + first_tick[0].admitted_count, + ACTION_BATCH_CANDIDATE_LIMIT_V1 + ); assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); assert_eq!( submission_ids @@ -3111,7 +3138,7 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { .is_some() }) .count(), - 64 + ACTION_BATCH_CANDIDATE_LIMIT_V1 ); let second_tick = host From 18fdeb8c395ada870d78a9ee905068cf0c75251e Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:17:44 -0700 Subject: [PATCH 46/64] Test: cover partitioned admission limits --- crates/warp-core/src/head_inbox.rs | 75 ++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index d0b13ec4..156826d8 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -1029,6 +1029,81 @@ mod tests { assert!(inbox.is_empty()); } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn partitioned_admission_applies_both_limit_directions_and_preserves_other_partition() { + for (policy_limit, partition_limit, expected_partition_count) in + [(2_u32, 5_usize, 2_usize), (5_u32, 3_usize, 3_usize)] + { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl( + u8::try_from(policy_limit).expect("the fixture policy limit fits in u8") + ), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::Budgeted { + max_per_tick: policy_limit, + }, + ); + let partition_kind = test_kind(); + for index in 0..6_u8 { + assert_eq!( + inbox.ingest(make_envelope( + partition_kind, + format!("partition-{policy_limit}-{index}").as_bytes(), + )), + InboxIngestResult::Accepted + ); + } + let other = (0..2_u8) + .map(|index| { + make_envelope( + other_kind(), + format!("other-{policy_limit}-{index}").as_bytes(), + ) + }) + .collect::>(); + let mut other_ids = other + .iter() + .map(IngressEnvelope::ingress_id) + .collect::>(); + other_ids.sort_unstable(); + for envelope in other { + assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); + } + + let partition_batch = inbox.admit_partitioned( + partition_kind, + partition_limit, + crate::WorldlineTick::ZERO, + ); + assert_eq!(partition_batch.len(), expected_partition_count); + assert!(partition_batch.iter().all(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ) + })); + + let other_batch = inbox.admit_partitioned( + partition_kind, + partition_limit, + crate::WorldlineTick::from_raw(1), + ); + assert_eq!( + other_batch + .iter() + .map(IngressEnvelope::ingress_id) + .collect::>(), + other_ids, + "the non-selected partition must remain byte-for-byte pending" + ); + assert_eq!(inbox.pending_count(), 6 - expected_partition_count); + } + } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] #[test] fn partitioned_admission_cannot_starve_other_category_by_ingress_hash() { From a12bd02d579712d7387b7dcb970b4b3a95689a23 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:19:39 -0700 Subject: [PATCH 47/64] Test: reach the Action blocker allocation guard --- crates/warp-core/src/echo_operation.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index c0960baf..d65180d3 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -6402,6 +6402,12 @@ mod tests { bytes.extend_from_slice(&digest(3)); bytes.extend_from_slice(&digest(4)); bytes.extend_from_slice(&digest(5)); + for budget_value in [1_u64, 2, 3] { + bytes.extend_from_slice(&budget_value.to_le_bytes()); + } + for identity_seed in 6..14 { + bytes.extend_from_slice(&digest(identity_seed)); + } bytes.extend_from_slice(&u64::MAX.to_le_bytes()); let error = recover_action_outcome_v1(&bytes) @@ -6410,6 +6416,7 @@ mod tests { error.kind(), EchoOperationArtifactErrorKindV1::InvalidStructure ); + assert_eq!(error.detail(), "Action blocker bytes are truncated"); } #[test] From c7c4d75692d5183685a2b167e9208b399ca55468 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:21:42 -0700 Subject: [PATCH 48/64] Fix: centralize extra test feature lanes --- scripts/verify-local.sh | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/scripts/verify-local.sh b/scripts/verify-local.sh index 25a15c0e..ef8fc503 100755 --- a/scripts/verify-local.sh +++ b/scripts/verify-local.sh @@ -1190,6 +1190,17 @@ pre_push_feature_string_for_test_target() { esac } +pre_push_extra_feature_strings_for_test_target() { + local crate="$1" + local test_target="$2" + + case "${crate}:${test_target}" in + warp-core:executable_operation_pipeline_tests) + printf '%s\n' "native_rule_bootstrap,trusted_runtime,host_test" + ;; + esac +} + append_pre_push_rust_slice() { local slice="$1" local slices_name="$2" @@ -1198,7 +1209,7 @@ append_pre_push_rust_slice() { collect_pre_push_rust_slices() { local -a slices=() - local file crate target filter features relative_test + local file crate target filter features extra_features relative_test while IFS= read -r file; do [[ -z "$file" ]] && continue @@ -1213,11 +1224,12 @@ collect_pre_push_rust_slices() { target="$(basename "$file" .rs)" features="$(pre_push_feature_string_for_test_target "$crate" "$target")" append_pre_push_rust_slice "${crate}|test|${target}|${features}|" slices - if [[ "${crate}:${target}" == "warp-core:executable_operation_pipeline_tests" ]]; then + while IFS= read -r extra_features; do + [[ -z "$extra_features" ]] && continue append_pre_push_rust_slice \ - "${crate}|test|${target}|native_rule_bootstrap,trusted_runtime,host_test|" \ + "${crate}|test|${target}|${extra_features}|" \ slices - fi + done < <(pre_push_extra_feature_strings_for_test_target "$crate" "$target") ;; crates/*/src/lib.rs) crate="$(printf '%s\n' "$file" | sed -n 's#^crates/\([^/]*\)/.*#\1#p')" @@ -1437,27 +1449,31 @@ warp_core_feature_args_for_test() { run_warp_core_test_target() { local lane="$1" local test_target="$2" + local extra_features local -a feature_args=() mapfile -t feature_args < <(warp_core_feature_args_for_test "$test_target") lane_cargo "$lane" test -p warp-core "${feature_args[@]}" --test "$test_target" - if [[ "$test_target" == "executable_operation_pipeline_tests" ]]; then + while IFS= read -r extra_features; do + [[ -z "$extra_features" ]] && continue lane_cargo "$lane" test -p warp-core \ - --features native_rule_bootstrap,trusted_runtime,host_test \ + --features "$extra_features" \ --test "$test_target" - fi + done < <(pre_push_extra_feature_strings_for_test_target "warp-core" "$test_target") } run_warp_core_clippy_test_target() { local lane="$1" local test_target="$2" + local extra_features local -a feature_args=() mapfile -t feature_args < <(warp_core_feature_args_for_test "$test_target") lane_cargo "$lane" clippy -p warp-core "${feature_args[@]}" --test "$test_target" -- -D warnings -D missing_docs - if [[ "$test_target" == "executable_operation_pipeline_tests" ]]; then + while IFS= read -r extra_features; do + [[ -z "$extra_features" ]] && continue lane_cargo "$lane" clippy -p warp-core \ - --features native_rule_bootstrap,trusted_runtime,host_test \ + --features "$extra_features" \ --test "$test_target" -- -D warnings -D missing_docs - fi + done < <(pre_push_extra_feature_strings_for_test_target "warp-core" "$test_target") } prepare_warp_wasm_scope() { From cf1a0b9dbce530331ea97df68343b02073d7e766 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:22:49 -0700 Subject: [PATCH 49/64] Test: decouple legacy Tick framing rejection --- crates/warp-core/src/trusted_runtime_host.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 9987d008..0df57f63 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -7247,19 +7247,7 @@ mod tests { receipt_ref: receipt.receipt_ref, causal_parent_receipts: Vec::new(), }; - let mut action_outcome = b"EOACT003".to_vec(); - action_outcome.extend_from_slice(&receipt.receipt_ref.submission_id); - action_outcome.extend_from_slice(&[45; 32]); - action_outcome.extend_from_slice(&[2, 1]); - for byte in 46..=48 { - action_outcome.extend_from_slice(&[byte; 32]); - } - action_outcome.extend_from_slice(&1_u64.to_le_bytes()); - action_outcome.extend_from_slice(&2_u64.to_le_bytes()); - action_outcome.extend_from_slice(&3_u64.to_le_bytes()); - for byte in 49..=51 { - action_outcome.extend_from_slice(&[byte; 32]); - } + let action_outcome = vec![0xa5]; let mut builder = wal.builder( WalTransactionKind::SchedulerTick, From 7be9f41fcf24e1ca7c4be5c840348a66ca060133 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:24:00 -0700 Subject: [PATCH 50/64] Docs: state the homogeneous Action batch invariant --- crates/warp-core/src/coordinator.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index b2ae43c9..8d8de371 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -4301,6 +4301,10 @@ impl SchedulerCoordinator { let parents = provenance.tip_ref(key.worldline_id)?.into_iter().collect(); #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + // `HeadInbox::admit_partitioned` guarantees that one admitted + // batch contains either executable-operation Actions or + // non-Action work, never both. Inspecting the first member is + // therefore a batch classification, not a positional guess. let executable_action_batch = admitted.first().is_some_and(|envelope| { crate::echo_operation::echo_operation_action_invocation_bytes_v1(envelope) .is_some() From 376a139aec6988d2c195b9012ae7d279409de4a7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:25:09 -0700 Subject: [PATCH 51/64] Docs: name Action and Tick WAL witnesses --- docs/topics/WAL.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index 1c28665c..23810ace 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -309,7 +309,17 @@ canonical package and invocation admission, authority and budget refusal, exact-basis noncommit, program-substitution refusal, same-host preparation authority, typed private-evaluation obstruction, deterministic duplicate-host results, append-only WAL codes, activation refusal for process-only -installations, and fresh-host installation and consequence recovery. +installations, accepted-Action recovery, scheduler-owned decided Ticks, and +fresh-host installation and consequence recovery. + +The Action/Tick witnesses to read first are: + +- `accepted_executable_action_recovers_pending_before_scheduler_evaluation` +- `scheduler_commits_two_independent_executable_actions_in_one_durable_tick` +- `scheduler_tick_construction_failure_publishes_no_action_state_or_receipt` +- `scheduler_wal_failure_rolls_back_and_requeues_action` +- `typed_action_obstruction_is_durable_and_contributes_no_mutation` +- `filesystem_wal_recovers_installed_meaning_consequence_and_typed_receipt` The host surface lives in `crates/warp-core/src/trusted_runtime_host.rs`, especially `TrustedRuntimeHost`, `TrustedRuntimeApp`, `TrustedRuntimeWal`, From caa83d9cfe767e6189fb177d203f3b151beb7657 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:27:54 -0700 Subject: [PATCH 52/64] Fix: bound Action admission by ingress order --- CHANGELOG.md | 2 ++ crates/warp-core/src/trusted_runtime_host.rs | 17 +++++++--- .../executable_operation_pipeline_tests.rs | 32 +++++++++++++------ 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfe8558f..385dbbbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,8 @@ The v1 scheduler Action-candidate ceiling is exported as `ACTION_BATCH_CANDIDATE_LIMIT_V1`; its acceptance witness now proves the complete limit with independent, non-conflicting node targets. + When more than that ceiling is pending, runtime admission selects the bounded + set by canonical ingress identity rather than submission identity. A composite receipt cannot validate outside its complete Action-batch context. Legacy operation recovery-index roots remain byte-compatible when no Action outcome exists. Direct operation diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 0df57f63..4ae8e652 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -1798,7 +1798,7 @@ impl TrustedRuntimeHost { }; let available = crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1 .saturating_sub(self.admitted_echo_operation_actions.len()); - let pending = self + let mut pending = self .pending_echo_operation_actions .iter() .filter(|submission_id| { @@ -1806,10 +1806,17 @@ impl TrustedRuntimeHost { .admitted_echo_operation_actions .contains_key(*submission_id) }) - .take(available) - .copied() - .collect::>(); - for submission_id in pending { + .map(|submission_id| { + let submission = self + .runtime + .witnessed_submission(submission_id) + .ok_or(RuntimeError::UnknownIntentSubmission(*submission_id))?; + Ok((submission.ingress_id, *submission_id)) + }) + .collect::, RuntimeError>>()?; + pending.sort_unstable(); + pending.truncate(available); + for (_, submission_id) in pending { let submission = self .runtime .witnessed_submission(&submission_id) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index af806c5c..7293ecd5 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3093,7 +3093,7 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { let installed = install_fixture_operation(&mut host); host.install_echo_operation_action_admission_policy_v1(invocation_policy()); - let mut submission_ids = Vec::new(); + let mut submissions = Vec::new(); for (index, node) in nodes.into_iter().enumerate() { let replacement = format!("bounded-after-{index}"); let invocation = action_invocation( @@ -3109,12 +3109,13 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { invocation, ) .expect("the bounded invocation becomes one canonical Action"); - submission_ids.push( - host.app() - .submit_intent_with_runtime_wal_ack(envelope) - .expect("the bounded Action is durable") - .submission_id, - ); + let ingress_id = envelope.ingress_id(); + let submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the bounded Action is durable") + .submission_id; + submissions.push((ingress_id, submission_id)); } assert_eq!( host.runtime().pending_witnessed_submission_count(), @@ -3131,15 +3132,26 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { ); assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); assert_eq!( - submission_ids + submissions .iter() - .filter(|submission_id| { + .filter(|(_, submission_id)| { host.echo_operation_action_outcome_v1(submission_id) .is_some() }) .count(), ACTION_BATCH_CANDIDATE_LIMIT_V1 ); + submissions.sort_unstable(); + assert!(submissions[..ACTION_BATCH_CANDIDATE_LIMIT_V1] + .iter() + .all(|(_, submission_id)| host + .echo_operation_action_outcome_v1(submission_id) + .is_some())); + assert!(submissions[ACTION_BATCH_CANDIDATE_LIMIT_V1..] + .iter() + .all(|(_, submission_id)| host + .echo_operation_action_outcome_v1(submission_id) + .is_none())); let second_tick = host .tick_once() @@ -3147,7 +3159,7 @@ fn scheduler_candidate_limit_leaves_excess_action_pending() { assert_eq!(second_tick.len(), 1); assert_eq!(second_tick[0].admitted_count, 1); assert_eq!(host.runtime().pending_witnessed_submission_count(), 0); - assert!(submission_ids.iter().all(|submission_id| host + assert!(submissions.iter().all(|(_, submission_id)| host .echo_operation_action_outcome_v1(submission_id) .is_some())); } From bdd86324d28e69cc49ad2fa3619c7f7d2e8bb624 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:41:41 -0700 Subject: [PATCH 53/64] Fix: bind noncommitted Actions to deciding Ticks --- CHANGELOG.md | 5 +- crates/warp-core/src/coordinator.rs | 1 + crates/warp-core/src/echo_operation.rs | 250 +++++++++++++++--- crates/warp-core/src/lib.rs | 4 +- crates/warp-core/src/trusted_runtime_host.rs | 53 +++- .../executable_operation_pipeline_tests.rs | 63 ++++- ...uler-owned-executable-operation-actions.md | 6 +- 7 files changed, 339 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 385dbbbf..6ab48126 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,10 @@ one exact parent coordinate and contribute to one composite Tick while retaining candidate-specific application-basis propositions and per-Action typed outcomes. Footprint conflicts name earlier applied members; evaluator - and composition-budget obstructions contribute no operations. + and composition-budget obstructions contribute no operations. Every + noncommitted typed outcome carries its deciding writer head, worldline and + global Tick coordinates, commit and Tick-receipt identities, and canonical + member index. One scheduler WAL transaction retains exactly one batched Tick decision record, then each Action's receipt correlation and typed outcome in canonical order, followed by exactly one replayable state delta. Recovery rejects every diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 8d8de371..cf66677b 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -4368,6 +4368,7 @@ impl SchedulerCoordinator { .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))?; crate::echo_operation::commit_scheduler_action_batch_to_state_v1( candidates, + *key, frontier.state_mut(), next_global_tick, engine.echo_operation_policy_id(), diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index d65180d3..2b7ae4b3 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -32,7 +32,7 @@ use crate::{ attachment::{AtomPayload, AttachmentKey, AttachmentValue}, clock::{GlobalTick, WorldlineTick}, footprint::{Footprint, WarpScopedPortKey}, - head::WriterHeadKey, + head::{HeadId, WriterHeadKey}, head_inbox::{make_intent_kind, IngressEnvelope, IngressPayload, IngressTarget, IntentKind}, ident::{EdgeKey, Hash, NodeKey, TypeId}, receipt::{TickReceipt, TickReceiptDisposition, TickReceiptEntry, TickReceiptRejection}, @@ -2747,6 +2747,7 @@ pub(crate) fn runtime_basis_obstruction( }), invocation_id: admitted.invocation_id, evaluation_basis_id: admitted.invocation.evaluation_basis.identity(), + decision_coordinate: None, }) } @@ -2789,6 +2790,56 @@ struct EchoOperationObstructionAdmissionEvidenceV1 { admission_id: EchoOperationInvocationAdmissionIdV1, } +/// Exact scheduler-owned Tick coordinate which decided one noncommitted Action +/// outcome. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EchoOperationActionDecisionCoordinateV1 { + writer_head: WriterHeadKey, + worldline_tick_after: WorldlineTick, + commit_global_tick: GlobalTick, + commit_id: Hash, + tick_receipt_digest: Hash, + member_index: u32, +} + +impl EchoOperationActionDecisionCoordinateV1 { + /// Returns the writer head whose scheduler Tick decided the Action. + #[must_use] + pub const fn writer_head(self) -> WriterHeadKey { + self.writer_head + } + + /// Returns the worldline coordinate after the deciding Tick. + #[must_use] + pub const fn worldline_tick_after(self) -> WorldlineTick { + self.worldline_tick_after + } + + /// Returns the runtime-global coordinate of the deciding Tick. + #[must_use] + pub const fn commit_global_tick(self) -> GlobalTick { + self.commit_global_tick + } + + /// Returns the deciding Tick's commit identity. + #[must_use] + pub const fn commit_id(self) -> Hash { + self.commit_id + } + + /// Returns the deciding Tick receipt's digest. + #[must_use] + pub const fn tick_receipt_digest(self) -> Hash { + self.tick_receipt_digest + } + + /// Returns this Action's canonical member index in the deciding Tick. + #[must_use] + pub const fn member_index(self) -> u32 { + self.member_index + } +} + /// One typed obstruction. Obstruction never carries a parent-visible patch. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EchoOperationObstructionV1 { @@ -2798,6 +2849,7 @@ pub struct EchoOperationObstructionV1 { invocation_admission: Box, invocation_id: EchoOperationInvocationIdV1, evaluation_basis_id: EchoOperationEvaluationBasisIdV1, + decision_coordinate: Option>, } impl EchoOperationObstructionV1 { @@ -2831,6 +2883,16 @@ impl EchoOperationObstructionV1 { self.evaluation_basis_id } + /// Returns the deciding Tick coordinate once this obstruction becomes a + /// terminal Action outcome. + #[must_use] + pub const fn decision_coordinate(&self) -> Option { + match &self.decision_coordinate { + Some(coordinate) => Some(**coordinate), + None => None, + } + } + /// Returns the exact runtime admission that authorized this evaluation attempt. #[must_use] pub const fn invocation_admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { @@ -2845,6 +2907,15 @@ impl EchoOperationObstructionV1 { self.invocation_admission.maximum_budget } + pub(crate) fn has_same_predecision_evidence(&self, other: &Self) -> bool { + self.kind == other.kind + && self.package_id == other.package_id + && self.installed_operation_id == other.installed_operation_id + && self.invocation_admission == other.invocation_admission + && self.invocation_id == other.invocation_id + && self.evaluation_basis_id == other.evaluation_basis_id + } + /// Returns the identity of this typed no-parent-patch obstruction. #[must_use] pub fn identity(&self) -> EchoOperationObstructionIdV1 { @@ -2890,6 +2961,8 @@ pub struct EchoOperationFootprintConflictV1 { pub(crate) preparation_id: PreparedEchoOperationIdV1, /// Canonical indices of earlier applied Tick members that blocked it. pub(crate) blocked_by: Vec, + /// Exact scheduler Tick and member position which decided the conflict. + pub(crate) decision_coordinate: Option>, } impl EchoOperationFootprintConflictV1 { @@ -2971,6 +3044,16 @@ impl EchoOperationFootprintConflictV1 { pub fn blocked_by(&self) -> &[u32] { &self.blocked_by } + + /// Returns the deciding Tick coordinate once this conflict becomes a + /// terminal Action outcome. + #[must_use] + pub const fn decision_coordinate(&self) -> Option { + match &self.decision_coordinate { + Some(coordinate) => Some(**coordinate), + None => None, + } + } } /// Typed terminal outcome for one executable-operation Action selected by a @@ -3010,6 +3093,20 @@ impl EchoOperationActionOutcomeV1 { conflict.blocked_by = replacement; } } + + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_decision_member_index_for_test(&mut self, replacement: u32) { + let coordinate = match self { + Self::Committed(_) => None, + Self::Obstructed(obstruction) => obstruction.decision_coordinate.as_deref_mut(), + Self::RejectedFootprintConflict(conflict) => { + conflict.decision_coordinate.as_deref_mut() + } + }; + if let Some(coordinate) = coordinate { + coordinate.member_index = replacement; + } + } } pub(crate) fn retain_action_outcome_v1( @@ -3060,6 +3157,12 @@ pub(crate) fn retain_action_outcome_v1( out.extend_from_slice(&obstruction.invocation_admission.admission_id.as_hash()); out.extend_from_slice(&obstruction.invocation_id.as_hash()); out.extend_from_slice(&obstruction.evaluation_basis_id.as_hash()); + retain_action_decision_coordinate_v1( + &mut out, + obstruction.decision_coordinate().ok_or_else(|| { + invalid_structure("terminal Action obstruction has no deciding Tick") + })?, + ); } EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { out.push(3); @@ -3092,6 +3195,12 @@ pub(crate) fn retain_action_outcome_v1( out.extend_from_slice(&conflict.prepared_result_id.as_hash()); out.extend_from_slice(&conflict.actual_footprint_digest); out.extend_from_slice(&conflict.preparation_id.as_hash()); + retain_action_decision_coordinate_v1( + &mut out, + conflict.decision_coordinate().ok_or_else(|| { + invalid_structure("terminal Action conflict has no deciding Tick") + })?, + ); let blocker_count = u64::try_from(conflict.blocked_by.len()) .map_err(|_| invalid_structure("Action blocker count is not representable"))?; out.extend_from_slice(&blocker_count.to_le_bytes()); @@ -3103,6 +3212,44 @@ pub(crate) fn retain_action_outcome_v1( Ok(out) } +fn retain_action_decision_coordinate_v1( + out: &mut Vec, + coordinate: EchoOperationActionDecisionCoordinateV1, +) { + out.extend_from_slice(coordinate.writer_head.worldline_id.as_bytes()); + out.extend_from_slice(coordinate.writer_head.head_id.as_bytes()); + out.extend_from_slice(&coordinate.worldline_tick_after.as_u64().to_le_bytes()); + out.extend_from_slice(&coordinate.commit_global_tick.as_u64().to_le_bytes()); + out.extend_from_slice(&coordinate.commit_id); + out.extend_from_slice(&coordinate.tick_receipt_digest); + out.extend_from_slice(&u64::from(coordinate.member_index).to_le_bytes()); +} + +fn recover_action_decision_coordinate_v1( + bytes: &[u8], + offset: &mut usize, +) -> Result { + let worldline_id = crate::WorldlineId::from_bytes(read_action_outcome_hash(bytes, offset)?); + let head_id = HeadId::from_bytes(read_action_outcome_hash(bytes, offset)?); + let worldline_tick_after = WorldlineTick::from_raw(read_action_outcome_u64(bytes, offset)?); + let commit_global_tick = GlobalTick::from_raw(read_action_outcome_u64(bytes, offset)?); + let commit_id = read_action_outcome_hash(bytes, offset)?; + let tick_receipt_digest = read_action_outcome_hash(bytes, offset)?; + let member_index = u32::try_from(read_action_outcome_u64(bytes, offset)?) + .map_err(|_| invalid_structure("Action Tick member index is not representable"))?; + Ok(EchoOperationActionDecisionCoordinateV1 { + writer_head: WriterHeadKey { + worldline_id, + head_id, + }, + worldline_tick_after, + commit_global_tick, + commit_id, + tick_receipt_digest, + member_index, + }) +} + pub(crate) fn recover_action_outcome_v1( bytes: &[u8], ) -> Result<(Hash, Hash, EchoOperationActionOutcomeV1), EchoOperationArtifactErrorV1> { @@ -3169,6 +3316,10 @@ pub(crate) fn recover_action_outcome_v1( bytes, &mut offset, )?), + decision_coordinate: Some(Box::new(recover_action_decision_coordinate_v1( + bytes, + &mut offset, + )?)), }) } 3 => { @@ -3196,6 +3347,7 @@ pub(crate) fn recover_action_outcome_v1( let actual_footprint_digest = read_action_outcome_hash(bytes, &mut offset)?; let preparation_id = PreparedEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let decision_coordinate = recover_action_decision_coordinate_v1(bytes, &mut offset)?; let blocker_count = read_action_outcome_u64(bytes, &mut offset)?; let maximum_encoded_blockers = u64::try_from(bytes.len().saturating_sub(offset) / core::mem::size_of::()) @@ -3233,6 +3385,7 @@ pub(crate) fn recover_action_outcome_v1( actual_footprint_digest, preparation_id, blocked_by, + decision_coordinate: Some(Box::new(decision_coordinate)), }, )) } @@ -3460,6 +3613,7 @@ pub(crate) fn prepare_operation_v1( }), invocation_id, evaluation_basis_id: admitted.invocation.evaluation_basis.identity(), + decision_coordinate: None, }) }; let Some(installed) = installed else { @@ -4591,6 +4745,7 @@ fn scheduler_composition_budget_obstruction_v1( }), invocation_id: prepared.invocation_id(), evaluation_basis_id: prepared.evaluation_basis().identity(), + decision_coordinate: None, } } @@ -4607,6 +4762,7 @@ struct EchoOperationTerminalMaterialV1 { pub(crate) fn commit_scheduler_action_batch_to_state_v1( mut candidates: Vec, + writer_head: WriterHeadKey, state: &mut WorldlineState, commit_global_tick: GlobalTick, policy_id: u32, @@ -4755,6 +4911,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( actual_footprint_digest: prepared.actual_footprint_digest(), preparation_id: prepared.preparation_id(), blocked_by: blockers, + decision_coordinate: None, }, )), )); @@ -4807,37 +4964,52 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( let outcomes = decisions .into_iter() - .map(|(submission_id, decision)| { - let outcome = match decision { - SchedulerEchoOperationDecisionV1::Applied(prepared) => { - let mut receipt = build_receipt( - &prepared, - EchoOperationTerminalMaterialV1 { - posture: EchoOperationTerminalPostureV1::Committed, - state_root_before, - state_root_after, - commit_id, - tick_receipt_digest: tick_receipt.digest(), - commit_global_tick: Some(commit_global_tick), - worldline_tick_after: WorldlineTick::from_raw(tx_raw), - }, - ); - receipt.committed_patch_digest = Some(patch.digest()); - receipt.composition_digest = Some(composition_digest); - receipt.terminal_outcome_digest = terminal_outcome_digest(&receipt); - receipt.receipt_digest = receipt_digest(&receipt); - EchoOperationActionOutcomeV1::Committed(Box::new(receipt)) - } - SchedulerEchoOperationDecisionV1::Obstructed(obstruction) => { - EchoOperationActionOutcomeV1::Obstructed(obstruction) - } - SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(conflict) => { - EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) - } - }; - (submission_id, outcome) - }) - .collect(); + .enumerate() + .map( + |(member_index, (submission_id, decision))| -> Result<_, EchoOperationCommitErrorV1> { + let member_index = u32::try_from(member_index) + .map_err(|_| EchoOperationCommitErrorV1::TooManyCandidates)?; + let decision_coordinate = EchoOperationActionDecisionCoordinateV1 { + writer_head, + worldline_tick_after: WorldlineTick::from_raw(tx_raw), + commit_global_tick, + commit_id, + tick_receipt_digest: tick_receipt.digest(), + member_index, + }; + let outcome = match decision { + SchedulerEchoOperationDecisionV1::Applied(prepared) => { + let mut receipt = build_receipt( + &prepared, + EchoOperationTerminalMaterialV1 { + posture: EchoOperationTerminalPostureV1::Committed, + state_root_before, + state_root_after, + commit_id, + tick_receipt_digest: tick_receipt.digest(), + commit_global_tick: Some(commit_global_tick), + worldline_tick_after: WorldlineTick::from_raw(tx_raw), + }, + ); + receipt.committed_patch_digest = Some(patch.digest()); + receipt.composition_digest = Some(composition_digest); + receipt.terminal_outcome_digest = terminal_outcome_digest(&receipt); + receipt.receipt_digest = receipt_digest(&receipt); + EchoOperationActionOutcomeV1::Committed(Box::new(receipt)) + } + SchedulerEchoOperationDecisionV1::Obstructed(mut obstruction) => { + obstruction.decision_coordinate = Some(Box::new(decision_coordinate)); + EchoOperationActionOutcomeV1::Obstructed(obstruction) + } + SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(mut conflict) => { + conflict.decision_coordinate = Some(Box::new(decision_coordinate)); + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) + } + }; + Ok((submission_id, outcome)) + }, + ) + .collect::, _>>()?; state.warp_state = next_state; state.last_snapshot = Some(snapshot.clone()); @@ -6408,6 +6580,20 @@ mod tests { for identity_seed in 6..14 { bytes.extend_from_slice(&digest(identity_seed)); } + retain_action_decision_coordinate_v1( + &mut bytes, + EchoOperationActionDecisionCoordinateV1 { + writer_head: WriterHeadKey { + worldline_id: crate::WorldlineId::from_bytes(digest(14)), + head_id: crate::make_head_id("blocker-count-guard"), + }, + worldline_tick_after: WorldlineTick::from_raw(1), + commit_global_tick: GlobalTick::from_raw(1), + commit_id: digest(15), + tick_receipt_digest: digest(16), + member_index: 0, + }, + ); bytes.extend_from_slice(&u64::MAX.to_le_bytes()); let error = recover_action_outcome_v1(&bytes) diff --git a/crates/warp-core/src/lib.rs b/crates/warp-core/src/lib.rs index c8406f29..88235660 100644 --- a/crates/warp-core/src/lib.rs +++ b/crates/warp-core/src/lib.rs @@ -241,8 +241,8 @@ pub use echo_operation::{ echo_operation_atom_value_digest_v1, echo_operation_create_if_absent_target_profile_identity_v1, echo_operation_package_id_v1, echo_operation_target_profile_identity_v1, AdmittedEchoOperationInvocationV1, - AdmittedExecutableOperationPackageV1, EchoOperationActionOutcomeV1, - EchoOperationAdmissionErrorKindV1, EchoOperationAdmissionErrorV1, + AdmittedExecutableOperationPackageV1, EchoOperationActionDecisionCoordinateV1, + EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorKindV1, EchoOperationAdmissionErrorV1, EchoOperationAdmissionPolicyV1, EchoOperationAnchoredNodeOccupancyV1, EchoOperationApplicationBasisV1, EchoOperationArtifactErrorKindV1, EchoOperationArtifactErrorV1, EchoOperationBudgetV1, EchoOperationCommitErrorV1, diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 4ae8e652..78faf593 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -589,6 +589,23 @@ impl TrustedRuntimeWalRecovery { } } + /// Replaces one noncommitted Action outcome's deciding member index for + /// adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_decision_member_index_for_test( + &mut self, + submission_id: Hash, + replacement: u32, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(retained_submission_id, _, _)| *retained_submission_id == submission_id) + { + outcome.replace_decision_member_index_for_test(replacement); + } + } + /// Re-runs activation-time Action parent-state checks for adversarial tests. #[cfg(any(test, feature = "host_test"))] pub fn validate_echo_operation_parent_states_for_test( @@ -4282,7 +4299,7 @@ fn validate_recovered_echo_operation_parent_states( )?; match &evaluation { EchoOperationPreparationV1::Obstructed(reconstructed) - if reconstructed == obstruction => {} + if reconstructed.has_same_predecision_evidence(obstruction) => {} EchoOperationPreparationV1::Prepared(_) if obstruction.kind() == crate::EchoOperationObstructionKindV1::BudgetExceeded => {} @@ -4393,6 +4410,7 @@ fn validate_recovered_echo_operation_parent_states( .collect::, _>>()?; let reconstructed_batch = commit_scheduler_action_batch_to_state_v1( candidates, + head_key, &mut reconstructed_state, commit_global_tick, policy_id, @@ -5105,21 +5123,39 @@ fn validate_recovered_echo_operation_action_outcomes( } } EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + let decision_coordinate_matches = + obstruction.decision_coordinate().is_some_and(|coordinate| { + action_decision_coordinate_matches_correlation_v1( + coordinate, + correlation, + index, + ) + }); if tick_entry.disposition != TickReceiptDisposition::Rejected( TickReceiptRejection::ExecutableOperationObstruction, ) || !blockers.is_empty() || tick_entry.rule_id != obstruction.installed_operation_id().as_hash() + || !decision_coordinate_matches { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } } EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + let decision_coordinate_matches = + conflict.decision_coordinate().is_some_and(|coordinate| { + action_decision_coordinate_matches_correlation_v1( + coordinate, + correlation, + index, + ) + }); if tick_entry.disposition != TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) || tick_entry.rule_id != conflict.installed_operation_id.as_hash() || blockers != conflict.blocked_by + || !decision_coordinate_matches { return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); } @@ -5130,6 +5166,21 @@ fn validate_recovered_echo_operation_action_outcomes( Ok(metrics) } +fn action_decision_coordinate_matches_correlation_v1( + coordinate: crate::EchoOperationActionDecisionCoordinateV1, + correlation: &ReceiptCorrelationPersistenceRecord, + member_index: usize, +) -> bool { + u32::try_from(member_index).is_ok_and(|member_index| { + coordinate.writer_head() == correlation.head_key + && coordinate.worldline_tick_after() == correlation.worldline_tick_after + && coordinate.commit_global_tick() == correlation.commit_global_tick + && coordinate.commit_id() == correlation.commit_hash + && coordinate.tick_receipt_digest() == correlation.tick_receipt_digest + && coordinate.member_index() == member_index + }) +} + fn wal_tick_decision_for_action_outcome(outcome: &EchoOperationActionOutcomeV1) -> WalTickDecision { match outcome { EchoOperationActionOutcomeV1::Committed(_) => WalTickDecision::Applied, diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 7293ecd5..2cf7adea 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3386,7 +3386,8 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { .submit_intent_with_runtime_wal_ack(envelope) .expect("the Action itself is durably accepted") .submission_id; - host.tick_once() + let steps = host + .tick_once() .expect("typed obstruction is a lawful scheduler decision"); let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = host.echo_operation_action_outcome_v1(&submission_id) @@ -3397,6 +3398,16 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { obstruction.kind(), EchoOperationObstructionKindV1::PreconditionMismatch ); + let decision = obstruction + .decision_coordinate() + .expect("the terminal obstruction names its deciding Tick"); + assert_eq!(decision.writer_head(), head_key); + assert_eq!( + decision.worldline_tick_after(), + steps[0].worldline_tick_after + ); + assert_eq!(decision.commit_id(), steps[0].commit_hash); + assert_eq!(decision.member_index(), 0); let state = host .runtime() .worldlines() @@ -3439,6 +3450,17 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { ), Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) )); + let mut forged_decision = host + .runtime_wal() + .expect("the obstruction fixture retains its WAL") + .recover_read_only() + .expect("the honest obstruction WAL recovers again"); + forged_decision + .replace_echo_operation_action_decision_member_index_for_test(submission_id, 1); + assert!(matches!( + forged_decision.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); } } @@ -3455,6 +3477,11 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { obstruction.kind(), EchoOperationObstructionKindV1::PreconditionMismatch ); + let recovered_decision = obstruction + .decision_coordinate() + .expect("recovered obstruction retains its deciding Tick"); + assert_eq!(recovered_decision.writer_head(), head_key); + assert_eq!(recovered_decision.member_index(), 0); let state = recovered .runtime() .worldlines() @@ -3806,6 +3833,22 @@ fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { ) }) .expect("exactly one Action is rejected by the earlier applied footprint"); + let Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) = + host.echo_operation_action_outcome_v1(&conflict_submission_id) + else { + panic!("the rejected Action retains typed conflict evidence"); + }; + let decision = conflict + .decision_coordinate() + .expect("the terminal conflict names its deciding Tick"); + assert_eq!(decision.writer_head(), head_key); + assert_eq!( + decision.worldline_tick_after(), + steps[0].worldline_tick_after + ); + assert_eq!(decision.commit_id(), steps[0].commit_hash); + assert_eq!(decision.member_index(), 1); + assert_eq!(conflict.blocked_by(), &[0]); #[cfg(feature = "host_test")] { @@ -3841,14 +3884,22 @@ fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { } } - let (mut recovered, _, _) = fixture_host(); + let (mut recovered, recovered_head, _) = fixture_host(); recovered .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) .expect("fresh-host recovery reconstructs the honest footprint conflict"); - assert!(matches!( - recovered.echo_operation_action_outcome_v1(&conflict_submission_id), - Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(_)) - )); + let Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) = + recovered.echo_operation_action_outcome_v1(&conflict_submission_id) + else { + panic!("fresh-host recovery restores the typed conflict"); + }; + assert_eq!( + conflict + .decision_coordinate() + .expect("recovered conflict retains its deciding Tick") + .writer_head(), + recovered_head + ); } #[cfg(feature = "host_test")] diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md index 4de30748..ab9edd3c 100644 --- a/docs/adr/0025-scheduler-owned-executable-operation-actions.md +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -131,7 +131,11 @@ the private successor in full. The generic Tick receipt classifies scheduler disposition. A separate typed executable-Action outcome binds the exact invocation, preparation or obstruction, Tick identity, and committed member consequence. Application -semantics are not inferred from a generic receipt label. +semantics are not inferred from a generic receipt label. Obstruction and +footprint-conflict outcomes retain the deciding writer head, worldline and +global Tick coordinates, commit and Tick-receipt identities, and their +canonical member index, so their evidence is interpretable without a separate +correlation lookup. ### One Tick is one WAL transaction From 022ebf7d3e1a25f61c195ef069a554385ac88f68 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:46:15 -0700 Subject: [PATCH 54/64] Fix: preserve Action outcomes across host parts --- CHANGELOG.md | 4 + crates/warp-core/src/lib.rs | 4 +- crates/warp-core/src/trusted_runtime_host.rs | 110 +++++++++++++++--- .../executable_operation_pipeline_tests.rs | 3 + docs/topics/WAL.md | 4 + 5 files changed, 106 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ab48126..a669da7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,10 @@ complete limit with independent, non-conflicting node targets. When more than that ceiling is pending, runtime admission selects the bounded set by canonical ingress identity rather than submission identity. + `TrustedRuntimeHost::into_parts` now returns an opaque + `TrustedRuntimeHostParts` value consumed by `from_parts`, preserving WAL, + authority, policy, pending admission, and every typed Action outcome across + host decomposition and reconstruction. A composite receipt cannot validate outside its complete Action-batch context. Legacy operation recovery-index roots remain byte-compatible when no Action outcome exists. Direct operation diff --git a/crates/warp-core/src/lib.rs b/crates/warp-core/src/lib.rs index 88235660..e3e058e8 100644 --- a/crates/warp-core/src/lib.rs +++ b/crates/warp-core/src/lib.rs @@ -480,8 +480,8 @@ pub use tick_patch::{ #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] pub use trusted_runtime_host::{ EvidenceCatalogPosture, RuntimeWalActivationGap, TrustedRuntimeApp, TrustedRuntimeHost, - TrustedRuntimeHostError, TrustedRuntimeHostRunReport, TrustedRuntimeWal, - TrustedRuntimeWalConfig, TrustedRuntimeWalError, TrustedRuntimeWalRecovery, + TrustedRuntimeHostError, TrustedRuntimeHostParts, TrustedRuntimeHostRunReport, + TrustedRuntimeWal, TrustedRuntimeWalConfig, TrustedRuntimeWalError, TrustedRuntimeWalRecovery, TrustedRuntimeWalStoreKind, WitnessedCausalAnchorAdmission, }; pub use tx::TxId; diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 78faf593..be9c25b1 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -805,6 +805,62 @@ pub struct TrustedRuntimeHost { BTreeMap, } +/// Owned decomposition of a [`TrustedRuntimeHost`]. +/// +/// Besides the runtime, provenance service, and engine, this value retains +/// host-owned WAL, authority, policy, pending-admission, and typed Action +/// outcome state. Passing it back to [`TrustedRuntimeHost::from_parts`] is an +/// identity-preserving host lifecycle operation. +pub struct TrustedRuntimeHostParts { + runtime: WorldlineRuntime, + provenance: ProvenanceService, + engine: Engine, + runtime_wal: Option, + causal_anchor_support_policy: Option, + echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1, + echo_operation_action_admission_policy: Option, + echo_operation_action_outcomes: BTreeMap, + pending_echo_operation_actions: BTreeSet, + admitted_echo_operation_actions: BTreeMap, + echo_operation_action_admission_obstructions: + BTreeMap, +} + +impl TrustedRuntimeHostParts { + /// Returns the owned runtime as read-only evidence. + #[must_use] + pub const fn runtime(&self) -> &WorldlineRuntime { + &self.runtime + } + + /// Returns the owned runtime for host-level reconfiguration. + pub const fn runtime_mut(&mut self) -> &mut WorldlineRuntime { + &mut self.runtime + } + + /// Returns the owned provenance service as read-only evidence. + #[must_use] + pub const fn provenance(&self) -> &ProvenanceService { + &self.provenance + } + + /// Returns the owned provenance service for host-level reconfiguration. + pub const fn provenance_mut(&mut self) -> &mut ProvenanceService { + &mut self.provenance + } + + /// Returns the owned execution engine as read-only evidence. + #[must_use] + pub const fn engine(&self) -> &Engine { + &self.engine + } + + /// Returns the owned execution engine for host-level reconfiguration. + pub const fn engine_mut(&mut self) -> &mut Engine { + &mut self.engine + } +} + fn pending_echo_operation_action_ids_v1( runtime: &WorldlineRuntime, decided: &BTreeMap, @@ -850,34 +906,54 @@ impl TrustedRuntimeHost { }) } - /// Builds a trusted host from already-initialized parts. + /// Rebuilds a trusted host from its identity-preserving owned parts. #[must_use] - pub fn from_parts( - runtime: WorldlineRuntime, - provenance: ProvenanceService, - engine: Engine, - ) -> Self { - let pending_echo_operation_actions = - pending_echo_operation_action_ids_v1(&runtime, &BTreeMap::new()); + pub fn from_parts(parts: TrustedRuntimeHostParts) -> Self { + let TrustedRuntimeHostParts { + runtime, + provenance, + engine, + runtime_wal, + causal_anchor_support_policy, + echo_operation_evaluation_authority, + echo_operation_action_admission_policy, + echo_operation_action_outcomes, + pending_echo_operation_actions, + admitted_echo_operation_actions, + echo_operation_action_admission_obstructions, + } = parts; Self { runtime, provenance, engine, - runtime_wal: None, - causal_anchor_support_policy: None, - echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), - echo_operation_action_admission_policy: None, - echo_operation_action_outcomes: BTreeMap::new(), + runtime_wal, + causal_anchor_support_policy, + echo_operation_evaluation_authority, + echo_operation_action_admission_policy, + echo_operation_action_outcomes, pending_echo_operation_actions, - admitted_echo_operation_actions: BTreeMap::new(), - echo_operation_action_admission_obstructions: BTreeMap::new(), + admitted_echo_operation_actions, + echo_operation_action_admission_obstructions, } } /// Consumes the host and returns owned runtime parts. #[must_use] - pub fn into_parts(self) -> (WorldlineRuntime, ProvenanceService, Engine) { - (self.runtime, self.provenance, self.engine) + pub fn into_parts(self) -> TrustedRuntimeHostParts { + TrustedRuntimeHostParts { + runtime: self.runtime, + provenance: self.provenance, + engine: self.engine, + runtime_wal: self.runtime_wal, + causal_anchor_support_policy: self.causal_anchor_support_policy, + echo_operation_evaluation_authority: self.echo_operation_evaluation_authority, + echo_operation_action_admission_policy: self.echo_operation_action_admission_policy, + echo_operation_action_outcomes: self.echo_operation_action_outcomes, + pending_echo_operation_actions: self.pending_echo_operation_actions, + admitted_echo_operation_actions: self.admitted_echo_operation_actions, + echo_operation_action_admission_obstructions: self + .echo_operation_action_admission_obstructions, + } } /// Returns the host-owned runtime as read-only evidence. diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 2cf7adea..4f93125d 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2843,6 +2843,7 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { let steps = host .tick_once() .expect("the scheduler privately evaluates and commits both Actions"); + let host = TrustedRuntimeHost::from_parts(host.into_parts()); assert_eq!(steps.len(), 1); assert_eq!(steps[0].admitted_count, 2); assert_eq!(steps[0].worldline_tick_after.as_u64(), 1); @@ -3389,6 +3390,7 @@ fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { let steps = host .tick_once() .expect("typed obstruction is a lawful scheduler decision"); + let host = TrustedRuntimeHost::from_parts(host.into_parts()); let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = host.echo_operation_action_outcome_v1(&submission_id) else { @@ -3823,6 +3825,7 @@ fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { let steps = host .tick_once() .expect("the scheduler lawfully classifies the footprint conflict"); + let host = TrustedRuntimeHost::from_parts(host.into_parts()); assert_eq!(steps.len(), 1); conflict_submission_id = submission_ids .into_iter() diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index 23810ace..34eaf1b4 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -325,6 +325,10 @@ The host surface lives in `crates/warp-core/src/trusted_runtime_host.rs`, especially `TrustedRuntimeHost`, `TrustedRuntimeApp`, `TrustedRuntimeWal`, `submit_intent_with_runtime_wal_ack(...)`, `admit_causal_anchor(...)`, `causal_anchor_by_id(...)`, and `recover_read_only()`. +`TrustedRuntimeHostParts` makes host decomposition identity-preserving: +`into_parts()` and `from_parts(...)` carry WAL, authority, policy, pending +admission, and typed Action outcomes together with runtime, provenance, and +engine ownership. Related current authority lives in `docs/topics/RuntimeAuthority.md`, `docs/architecture/continuum-transport.md`, and From 3459337b75e1f54ca7494917bbba156355531ff5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 20:55:20 -0700 Subject: [PATCH 55/64] Fix: pin Action member alignment --- CHANGELOG.md | 3 ++- crates/warp-core/src/echo_operation.rs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a669da7a..4599f4de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,8 @@ and composition-budget obstructions contribute no operations. Every noncommitted typed outcome carries its deciding writer head, worldline and global Tick coordinates, commit and Tick-receipt identities, and canonical - member index. + member index. Tick construction pins that index to the corresponding receipt + entry with an executable alignment invariant. One scheduler WAL transaction retains exactly one batched Tick decision record, then each Action's receipt correlation and typed outcome in canonical order, followed by exactly one replayable state delta. Recovery rejects every diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 2b7ae4b3..73df1f60 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -4962,6 +4962,7 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( tx, }; + debug_assert_action_member_alignment_v1(decisions.len(), tick_receipt.entries().len()); let outcomes = decisions .into_iter() .enumerate() @@ -5026,6 +5027,13 @@ pub(crate) fn commit_scheduler_action_batch_to_state_v1( }) } +fn debug_assert_action_member_alignment_v1(decision_count: usize, receipt_entry_count: usize) { + debug_assert_eq!( + decision_count, receipt_entry_count, + "Action member index is the Tick receipt entry index" + ); +} + pub(crate) fn action_batch_patch_from_preparations_v1( policy_id: u32, ordered_rule_ids: &[Hash], @@ -6113,6 +6121,13 @@ mod tests { } } + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "Action member index is the Tick receipt entry index")] + fn action_member_alignment_rejects_divergent_decision_and_receipt_counts() { + debug_assert_action_member_alignment_v1(2, 1); + } + #[test] fn footprint_digest_is_prefix_free_across_adjacent_sets() { let mut read_node_bytes = [0; 64]; From 4e470837824cc75fca53d4d7bd90d83d81acebb4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:00:05 -0700 Subject: [PATCH 56/64] Fix: count Action installation lookups --- CHANGELOG.md | 4 +- crates/warp-core/src/trusted_runtime_host.rs | 51 +++++++++---------- .../executable_operation_pipeline_tests.rs | 15 +++--- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4599f4de..628a985b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,9 @@ Activation caches each reconstructed causal state by worldline coordinate, so Actions sharing one scheduler basis do not replay that history repeatedly. Recovered Action outcomes resolve installed packages through one package-ID - index instead of scanning the complete installation set per Action. + index instead of scanning the complete installation set per Action. Test + instrumentation now counts the actual package-index and installation-order + lookups rather than asserting constant zero-value proxies. The v1 scheduler Action-candidate ceiling is exported as `ACTION_BATCH_CANDIDATE_LIMIT_V1`; its acceptance witness now proves the complete limit with independent, non-conflicting node targets. diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index be9c25b1..971a4df2 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -436,7 +436,9 @@ struct RecoveredEchoOperationActionInstallationOrder { #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] struct RecoveredEchoOperationActionOutcomeValidationMetrics { #[cfg(any(test, feature = "host_test"))] - linear_installation_comparisons: usize, + indexed_package_lookups: usize, + #[cfg(any(test, feature = "host_test"))] + installation_order_lookups: usize, } /// Read-only runtime WAL recovery report. @@ -469,8 +471,6 @@ pub struct TrustedRuntimeWalRecovery { pub echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, echo_operation_action_decisions: BTreeMap, echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder, - #[cfg(any(test, feature = "host_test"))] - echo_operation_action_installation_snapshot_count: usize, causal_history_frontiers: Vec, } @@ -493,12 +493,12 @@ impl TrustedRuntimeWalRecovery { .map(|_| ()) } - /// Returns the number of linear installed-package comparisons used while - /// validating recovered Action outcomes. + /// Returns the indexed package and installation-order lookup counts used + /// while validating recovered Action outcomes. #[cfg(any(test, feature = "host_test"))] - pub fn echo_operation_action_linear_installation_comparisons_for_test( + pub fn echo_operation_action_installation_lookup_counts_for_test( &self, - ) -> Result { + ) -> Result<(usize, usize), TrustedRuntimeWalError> { validate_recovered_echo_operation_action_outcomes( &self.witnessed_submissions, &self.receipt_correlations, @@ -508,7 +508,12 @@ impl TrustedRuntimeWalRecovery { &self.echo_operation_action_decisions, &self.echo_operation_action_installation_order, ) - .map(|metrics| metrics.linear_installation_comparisons) + .map(|metrics| { + ( + metrics.indexed_package_lookups, + metrics.installation_order_lookups, + ) + }) } /// Replaces one recovered scheduler decision for adversarial recovery tests. @@ -533,14 +538,6 @@ impl TrustedRuntimeWalRecovery { .remove(&submission_id); } - /// Returns the number of per-Action installation-set snapshots constructed - /// during recovery. - #[cfg(any(test, feature = "host_test"))] - #[must_use] - pub fn echo_operation_action_installation_snapshot_count_for_test(&self) -> usize { - self.echo_operation_action_installation_snapshot_count - } - /// Replaces one retained obstruction kind for adversarial recovery tests. #[cfg(any(test, feature = "host_test"))] pub fn replace_echo_operation_action_obstruction_kind_for_test( @@ -2547,9 +2544,6 @@ impl TrustedRuntimeWal { let echo_operation_action_decisions = runtime_state.echo_operation_action_decisions; let echo_operation_action_installation_order = runtime_state.echo_operation_action_installation_order; - #[cfg(any(test, feature = "host_test"))] - let echo_operation_action_installation_snapshot_count = - runtime_state.echo_operation_action_installation_snapshot_count; let missing_runtime_state_deltas = runtime_state.missing_runtime_state_deltas; let installed_echo_operations = operation_material.installations; let echo_operation_receipts = operation_material.receipts; @@ -2593,8 +2587,6 @@ impl TrustedRuntimeWal { echo_operation_action_outcomes, echo_operation_action_decisions, echo_operation_action_installation_order, - #[cfg(any(test, feature = "host_test"))] - echo_operation_action_installation_snapshot_count, causal_history_frontiers, }) } @@ -4740,8 +4732,6 @@ struct RecoveredRuntimeStateMaterial { echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, echo_operation_action_decisions: BTreeMap, echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder, - #[cfg(any(test, feature = "host_test"))] - echo_operation_action_installation_snapshot_count: usize, missing_runtime_state_deltas: Vec, } @@ -4753,8 +4743,6 @@ fn recover_runtime_state_delta_material( let mut action_outcomes_by_submission = BTreeMap::new(); let mut action_decisions_by_submission = BTreeMap::new(); let mut action_installation_count_before_tick = BTreeMap::new(); - #[cfg(any(test, feature = "host_test"))] - let action_installation_snapshot_count = 0_usize; let mut installation_ordinals = BTreeMap::new(); let mut submission_by_ticket = BTreeMap::new(); let mut missing = Vec::new(); @@ -4919,8 +4907,6 @@ fn recover_runtime_state_delta_material( package_ordinals: installation_ordinals, installation_count_before_tick: action_installation_count_before_tick, }, - #[cfg(any(test, feature = "host_test"))] - echo_operation_action_installation_snapshot_count: action_installation_snapshot_count, missing_runtime_state_deltas: missing, }) } @@ -4959,7 +4945,8 @@ fn validate_recovered_echo_operation_action_outcomes( decisions: &BTreeMap, installation_order: &RecoveredEchoOperationActionInstallationOrder, ) -> Result { - let metrics = RecoveredEchoOperationActionOutcomeValidationMetrics::default(); + #[allow(unused_mut)] + let mut metrics = RecoveredEchoOperationActionOutcomeValidationMetrics::default(); let envelopes = witnessed_submissions .records() .iter() @@ -5023,9 +5010,17 @@ fn validate_recovered_echo_operation_action_outcomes( .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; let invocation = inspect_action_invocation_v1(invocation_bytes) .map_err(|_| TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + #[cfg(any(test, feature = "host_test"))] + { + metrics.indexed_package_lookups += 1; + } let installed = installed_by_package .get(&invocation.package_id) .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + #[cfg(any(test, feature = "host_test"))] + { + metrics.installation_order_lookups += 2; + } let installed_before_tick = installation_order .installation_count_before_tick .get(submission_id) diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 4f93125d..cd95c2b9 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -2930,17 +2930,16 @@ fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { let mut adversarial = runtime_wal .recover_read_only() .expect("the honest composite Tick recovers"); + let (indexed_package_lookups, installation_order_lookups) = adversarial + .echo_operation_action_installation_lookup_counts_for_test() + .expect("the honest composite Tick outcomes validate"); assert_eq!( - adversarial.echo_operation_action_installation_snapshot_count_for_test(), - 0, - "recovery must not clone the installed-package set per Action" + indexed_package_lookups, 2, + "recovery must perform one indexed package lookup per Action" ); assert_eq!( - adversarial - .echo_operation_action_linear_installation_comparisons_for_test() - .expect("the honest composite Tick outcomes validate"), - 0, - "recovery must resolve installed packages through one identity index" + installation_order_lookups, 4, + "recovery must perform two indexed installation-order lookups per Action" ); assert_eq!( adversarial From 434a6d9d61567c0a0f25b8bea0f657afc2c9b5de Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:01:18 -0700 Subject: [PATCH 57/64] Docs: distinguish rejected Action consequences --- ...-scheduler-owned-executable-operation-actions.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md index ab9edd3c..88829b11 100644 --- a/docs/adr/0025-scheduler-owned-executable-operation-actions.md +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -130,12 +130,13 @@ the private successor in full. The generic Tick receipt classifies scheduler disposition. A separate typed executable-Action outcome binds the exact invocation, preparation or -obstruction, Tick identity, and committed member consequence. Application -semantics are not inferred from a generic receipt label. Obstruction and -footprint-conflict outcomes retain the deciding writer head, worldline and -global Tick coordinates, commit and Tick-receipt identities, and their -canonical member index, so their evidence is interpretable without a separate -correlation lookup. +obstruction and Tick identity. Applied Actions additionally bind their +committed patch member; obstruction and footprint-conflict outcomes explicitly +carry no committed consequence. Application semantics are not inferred from a +generic receipt label. Rejected outcomes retain the deciding writer head, +worldline and global Tick coordinates, commit and Tick-receipt identities, and +their canonical member index, so their evidence is interpretable without a +separate correlation lookup. ### One Tick is one WAL transaction From ec83d177153854d5769031ee39a523ff6f4babd3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:10:27 -0700 Subject: [PATCH 58/64] Fix: scope Action admission by runnable head --- CHANGELOG.md | 4 +- crates/warp-core/src/trusted_runtime_host.rs | 34 ++++++- .../executable_operation_pipeline_tests.rs | 92 ++++++++++++++++++- 3 files changed, 123 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 628a985b..b0f7118b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,7 +60,9 @@ lookups rather than asserting constant zero-value proxies. The v1 scheduler Action-candidate ceiling is exported as `ACTION_BATCH_CANDIDATE_LIMIT_V1`; its acceptance witness now proves the - complete limit with independent, non-conflicting node targets. + complete limit with independent, non-conflicting node targets. Admission + applies that ceiling independently to each runnable head, so Actions retained + for dormant or faulted heads cannot consume another head's Tick capacity. When more than that ceiling is pending, runtime admission selects the bounded set by canonical ingress identity rather than submission identity. `TrustedRuntimeHost::into_parts` now returns an opaque diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 971a4df2..ddccc1b4 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -1886,8 +1886,24 @@ impl TrustedRuntimeHost { let Some(policy) = self.echo_operation_action_admission_policy else { return Ok(self.admitted_echo_operation_actions.clone()); }; - let available = crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1 - .saturating_sub(self.admitted_echo_operation_actions.len()); + let mut available_by_head = SchedulerCoordinator::peek_order(&self.runtime) + .into_iter() + .map(|head_key| { + ( + head_key, + crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1, + ) + }) + .collect::>(); + for submission_id in self.admitted_echo_operation_actions.keys() { + let submission = self + .runtime + .witnessed_submission(submission_id) + .ok_or(RuntimeError::UnknownIntentSubmission(*submission_id))?; + if let Some(available) = available_by_head.get_mut(&submission.head_key) { + *available = available.saturating_sub(1); + } + } let mut pending = self .pending_echo_operation_actions .iter() @@ -1901,11 +1917,21 @@ impl TrustedRuntimeHost { .runtime .witnessed_submission(submission_id) .ok_or(RuntimeError::UnknownIntentSubmission(*submission_id))?; - Ok((submission.ingress_id, *submission_id)) + Ok((submission.ingress_id, *submission_id, submission.head_key)) }) .collect::, RuntimeError>>()?; pending.sort_unstable(); - pending.truncate(available); + let pending = pending + .into_iter() + .filter_map(|(ingress_id, submission_id, head_key)| { + let available = available_by_head.get_mut(&head_key)?; + if *available == 0 { + return None; + } + *available -= 1; + Some((ingress_id, submission_id)) + }) + .collect::>(); for (_, submission_id) in pending { let submission = self .runtime diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index cd95c2b9..0284dc2e 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -28,8 +28,8 @@ use warp_core::{ EchoOperationInvocationAdmissionErrorKindV1, EchoOperationInvocationAdmissionPolicyV1, EchoOperationInvocationV1, EchoOperationObstructionKindV1, EchoOperationPreparationV1, EchoOperationProgramV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, - EngineBuilder, ExecutableOperationPackageV1, GraphStore, InboxPolicy, IngressTarget, - InstalledEchoOperationV1, NodeKey, NodeRecord, PlaybackMode, RuntimeError, + EngineBuilder, ExecutableOperationPackageV1, GraphStore, HeadEligibility, InboxPolicy, + IngressTarget, InstalledEchoOperationV1, NodeKey, NodeRecord, PlaybackMode, RuntimeError, RuntimeWalActivationGap, SchedulerKind, TrustedRuntimeHost, TrustedRuntimeHostError, TrustedRuntimeWalConfig, TrustedRuntimeWalError, WorldlineId, WorldlineRuntime, WorldlineState, WriterHead, WriterHeadKey, ACTION_BATCH_CANDIDATE_LIMIT_V1, @@ -3904,6 +3904,94 @@ fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { ); } +#[test] +fn dormant_head_actions_do_not_consume_runnable_head_admission_capacity() { + let attachment_type = make_type_id("operation-fixture-atom"); + let (mut host, runnable_head, runnable_node, dormant_node) = fixture_host_with_bare_node( + make_node_id("operation-fixture-dormant-node"), + make_type_id("operation-fixture-node"), + Some((attachment_type, b"dormant-before")), + ); + let dormant_head = WriterHeadKey { + worldline_id: runnable_head.worldline_id, + head_id: make_head_id("operation-fixture-dormant-writer"), + }; + let mut parts = host.into_parts(); + parts + .runtime_mut() + .register_writer_head(WriterHead::with_routing( + dormant_head, + PlaybackMode::Play, + InboxPolicy::AcceptAll, + None, + false, + )) + .expect("the dormant sibling head registers"); + host = TrustedRuntimeHost::from_parts(parts); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + for ordinal in 0..ACTION_BATCH_CANDIDATE_LIMIT_V1 { + let replacement = format!("dormant-after-{ordinal}"); + let invocation = action_invocation( + &host, + &installed, + dormant_head, + dormant_node, + b"dormant-before", + replacement.as_bytes(), + ); + host.app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: dormant_head }, + invocation, + ) + .expect("the dormant invocation is one canonical Action"), + ) + .expect("the dormant Action is accepted"); + } + let mut parts = host.into_parts(); + parts + .runtime_mut() + .set_head_eligibility(dormant_head, HeadEligibility::Dormant) + .expect("the sibling head becomes dormant before admission"); + host = TrustedRuntimeHost::from_parts(parts); + assert!(host + .tick_once() + .expect("dormant Actions do not execute") + .is_empty()); + + let runnable_invocation = action_invocation( + &host, + &installed, + runnable_head, + runnable_node, + b"before", + b"runnable-after", + ); + let runnable_submission_id = host + .app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: runnable_head }, + runnable_invocation, + ) + .expect("the runnable invocation is one canonical Action"), + ) + .expect("the runnable Action is accepted") + .submission_id; + + let steps = host + .tick_once() + .expect("the runnable Action is not starved by the dormant head"); + assert_eq!(steps.len(), 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&runnable_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + #[cfg(feature = "host_test")] #[test] fn scheduler_wal_failure_rolls_back_and_requeues_action() { From 49fc0fa3afae0fe61a153e5bd6e35f2476b11800 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:13:44 -0700 Subject: [PATCH 59/64] Fix: recover Actions on the deciding worldline --- CHANGELOG.md | 5 +- crates/warp-core/src/trusted_runtime_host.rs | 14 ++-- .../executable_operation_pipeline_tests.rs | 69 +++++++++++++++++++ 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0f7118b..789c9b6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,10 @@ consequence, exact reconstructed aggregate patch membership, and state root. Typed obstruction records retain their invocation-admission policy and budget ceiling; recovery reproduces bounded evaluation and the complete - scheduler composition before accepting an obstruction kind. + scheduler composition before accepting an obstruction kind. For a + cross-worldline basis obstruction, recovery reconstructs the submitted basis + on its named worldline but resolves the deciding transition on the target + head's worldline. Recovery records one monotonic installation ordinal per package and one installation-count boundary per Action, avoiding per-outcome package-set snapshots while preserving exact installation-before-Tick validation. diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index ddccc1b4..4554fd38 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -4235,8 +4235,8 @@ fn validate_recovered_echo_operation_parent_states( }, )?; let basis = invocation.evaluation_basis; - let worldline_id = basis.writer_head().worldline_id; - let frontier = runtime.worldlines().get(&worldline_id).ok_or( + let basis_worldline_id = basis.writer_head().worldline_id; + let frontier = runtime.worldlines().get(&basis_worldline_id).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action basis names an unavailable recovery worldline", }, @@ -4244,7 +4244,7 @@ fn validate_recovered_echo_operation_parent_states( let basis_state = recovered_worldline_state_at( &mut recovered_states, recovered_provenance, - worldline_id, + basis_worldline_id, frontier.state(), basis.worldline_tick(), "Action basis state cannot be reconstructed", @@ -4278,11 +4278,11 @@ fn validate_recovered_echo_operation_parent_states( detail: "Action basis posture disagrees with the scheduler Tick parent", }); } - let transition = entries.get(&(worldline_id, tick_before)).ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { + let transition = entries + .get(&(correlation.head_key.worldline_id, tick_before)) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action outcome has no recovered scheduler transition", - }, - )?; + })?; let policy_id = transition .patch .as_ref() diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 0284dc2e..eb2689bd 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3746,6 +3746,75 @@ fn identical_action_payloads_on_distinct_heads_keep_distinct_outcomes() { ); } +#[test] +fn cross_worldline_basis_obstruction_recovers_on_the_deciding_worldline() { + let wal_dir = TempWalDir::new(); + let submission_id; + + { + let (mut host, basis_head, target_head, node) = fixture_host_with_two_worldlines(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the cross-worldline obstruction WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let invocation = + action_invocation(&host, &installed, basis_head, node, b"before", b"after"); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: target_head }, + invocation, + ) + .expect("the cross-worldline invocation becomes one Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the cross-worldline Action is durable") + .submission_id; + + let steps = host + .tick_once() + .expect("the target worldline decides the basis obstruction"); + assert_eq!(steps.len(), 1); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("the cross-worldline Action retains one typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); + assert_eq!( + obstruction + .decision_coordinate() + .expect("the obstruction names its deciding Tick") + .writer_head(), + target_head + ); + } + + let (mut recovered, _, recovered_target_head, _) = fixture_host_with_two_worldlines(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("fresh-host recovery resolves the target-worldline transition"); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + recovered.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("fresh-host recovery restores the cross-worldline obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); + assert_eq!( + obstruction + .decision_coordinate() + .expect("the recovered obstruction names its deciding Tick") + .writer_head(), + recovered_target_head + ); +} + #[test] fn recovery_separates_identical_commits_on_distinct_worldlines() { let (mut host, first_head, second_head, node) = fixture_host_with_two_worldlines(); From 4086a353b41088395929fd01e7c7e3c55b465bed Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:19:42 -0700 Subject: [PATCH 60/64] Fix: alternate scheduler categories per round --- CHANGELOG.md | 5 +- crates/warp-core/src/coordinator.rs | 8 +- crates/warp-core/src/head_inbox.rs | 77 +++++++++++++++---- ...uler-owned-executable-operation-actions.md | 14 ++-- docs/spec/scheduler-warp-core.md | 12 +-- 5 files changed, 81 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 789c9b6e..c66aaa54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,9 @@ A WAL-enabled app surface rejects executable Actions submitted outside the durable acknowledgement boundary before witnessed intake can mutate. Mixed executable and provider/native backlogs alternate by durable parent - worldline-Tick parity, preventing caller-controlled ingress hashes from - starving either execution category. + global-Tick parity. The scheduler-round coordinate advances once per pass, so + every head switches categories even when several heads share one worldline, + preventing caller-controlled ingress hashes from starving either category. Positional receipt attribution is exclusive to executable Actions; ordinary correlations without an exact scope match fail closed. Scheduler selection admits at most 64 executable Actions per Tick, leaving diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index cf66677b..27054cc4 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -4266,11 +4266,7 @@ impl SchedulerCoordinator { for key in &keys { #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] - let partition_parent_tick = runtime - .worldlines - .get(&key.worldline_id) - .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))? - .frontier_tick(); + let partition_parent_global_tick = runtime.global_tick; let inbox = runtime .heads .inbox_mut(key) @@ -4280,7 +4276,7 @@ impl SchedulerCoordinator { inbox.admit_partitioned( crate::echo_operation::echo_operation_action_intent_kind_v1(), crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1, - partition_parent_tick, + partition_parent_global_tick, ) } else { inbox.admit() diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index 156826d8..597753b3 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -794,18 +794,19 @@ impl HeadInbox { /// Admits one deterministic execution category without mixing it with /// other pending categories. /// - /// When both categories are pending, `parent_worldline_tick` parity chooses + /// When both categories are pending, `parent_global_tick` parity chooses /// whether this batch contains the supplied `partition_kind` or everything - /// else. Using the durable parent coordinate means neither category can - /// starve and recovery does not depend on process-local state. When only one - /// category is pending, it proceeds immediately. Existing per-Tick limits - /// still bound the selected category. + /// else. The durable scheduler-round coordinate advances once per scheduler + /// pass, so every participating head alternates categories even when several + /// heads advance one shared worldline. Recovery does not depend on + /// process-local state. When only one category is pending, it proceeds + /// immediately. Existing per-Tick limits still bound the selected category. #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] pub(crate) fn admit_partitioned( &mut self, partition_kind: IntentKind, partition_limit: usize, - parent_worldline_tick: crate::WorldlineTick, + parent_global_tick: crate::GlobalTick, ) -> Vec { if self.pending.is_empty() { return Vec::new(); @@ -825,7 +826,7 @@ impl HeadInbox { } } let selected_partition = match (has_partition, has_other) { - (true, true) => parent_worldline_tick.as_u64().is_multiple_of(2), + (true, true) => parent_global_tick.as_u64().is_multiple_of(2), (true, false) => true, (false, true) => false, (false, false) => return Vec::new(), @@ -1002,7 +1003,7 @@ mod tests { assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); } - let first_tick = crate::WorldlineTick::from_raw(u64::from(!first_is_partition)); + let first_tick = crate::GlobalTick::from_raw(u64::from(!first_is_partition)); let first_batch = inbox.admit_partitioned(partition_kind, 2, first_tick); assert_eq!(first_batch.len(), 2); assert!(first_batch.iter().all(|envelope| { @@ -1029,6 +1030,55 @@ mod tests { assert!(inbox.is_empty()); } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn scheduler_round_alternates_each_head_despite_even_worldline_progress() { + let partition_kind = test_kind(); + for head_label in ["first", "second"] { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id(head_label), + }, + InboxPolicy::Budgeted { max_per_tick: 1 }, + ); + for index in 0..2_u8 { + assert_eq!( + inbox.ingest(make_envelope( + partition_kind, + format!("{head_label}-partition-{index}").as_bytes(), + )), + InboxIngestResult::Accepted + ); + assert_eq!( + inbox.ingest(make_envelope( + other_kind(), + format!("{head_label}-other-{index}").as_bytes(), + )), + InboxIngestResult::Accepted + ); + } + + let first_batch = inbox.admit_partitioned(partition_kind, 1, crate::GlobalTick::ZERO); + let second_batch = + inbox.admit_partitioned(partition_kind, 1, crate::GlobalTick::from_raw(1)); + let is_partition = |envelope: &IngressEnvelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ) + }; + assert_eq!(first_batch.len(), 1); + assert_eq!(second_batch.len(), 1); + assert_ne!( + is_partition(&first_batch[0]), + is_partition(&second_batch[0]), + "each head must switch categories on the next scheduler round" + ); + } + } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] #[test] fn partitioned_admission_applies_both_limit_directions_and_preserves_other_partition() { @@ -1073,11 +1123,8 @@ mod tests { assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); } - let partition_batch = inbox.admit_partitioned( - partition_kind, - partition_limit, - crate::WorldlineTick::ZERO, - ); + let partition_batch = + inbox.admit_partitioned(partition_kind, partition_limit, crate::GlobalTick::ZERO); assert_eq!(partition_batch.len(), expected_partition_count); assert!(partition_batch.iter().all(|envelope| { matches!( @@ -1090,7 +1137,7 @@ mod tests { let other_batch = inbox.admit_partitioned( partition_kind, partition_limit, - crate::WorldlineTick::from_raw(1), + crate::GlobalTick::from_raw(1), ); assert_eq!( other_batch @@ -1137,7 +1184,7 @@ mod tests { let batch = inbox.admit_partitioned( partition_kind, 1, - crate::WorldlineTick::from_raw(u64::from(round)), + crate::GlobalTick::from_raw(u64::from(round)), ); assert_eq!(batch.len(), 1); legacy_admitted |= batch[0].ingress_id() == legacy_ingress_id; diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md index 88829b11..0a5e0414 100644 --- a/docs/adr/0025-scheduler-owned-executable-operation-actions.md +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -85,14 +85,16 @@ Provider/native ingress and executable-operation Actions are different execution categories. A head inbox may contain both while migrations are in flight, but one Tick candidate batch is homogeneous. -When both categories are pending, the exact parent worldline-Tick parity selects -the category: an even parent Tick selects executable Actions and an odd parent -Tick selects provider/native ingress. When only one category is pending, that +When both categories are pending, the exact parent global-Tick parity selects +the category: an even scheduler round selects executable Actions and an odd +round selects provider/native ingress. When only one category is pending, that category proceeds immediately. Entries remain canonically ordered by ingress identity inside the selected category and existing inbox budgets still apply. -This durable alternating choice prevents caller-controlled hashes from starving -one evaluator category, remains stable across restart, and makes it impossible -for an executable Action to fall through to a native callback engine. +The durable global coordinate advances once per scheduler pass, so every head +alternates even when several heads advance one shared worldline. This choice +prevents caller-controlled hashes from starving one evaluator category, remains +stable across restart, and makes it impossible for an executable Action to fall +through to a native callback engine. ### One exact parent basis, per-Action application propositions diff --git a/docs/spec/scheduler-warp-core.md b/docs/spec/scheduler-warp-core.md index 26e6c4c2..b2471b03 100644 --- a/docs/spec/scheduler-warp-core.md +++ b/docs/spec/scheduler-warp-core.md @@ -69,12 +69,12 @@ application semantics. The scheduler partitions executable and legacy/native ingress deterministically so the two evaluator categories never share one candidate batch. When both are -pending, exact parent worldline-Tick parity alternates the selected category; -when only one is pending, it proceeds immediately. Inside either category, -ingress order is canonical. Private bounded evaluation yields either a complete -prepared candidate or a typed no-mutation obstruction. The scheduler reserves -successful candidate footprints, constructs one composite consequence, and -emits one Tick receipt entry per Action. +pending, exact parent global-Tick parity alternates the selected category for +every head on each scheduler pass; when only one is pending, it proceeds +immediately. Inside either category, ingress order is canonical. Private bounded +evaluation yields either a complete prepared candidate or a typed no-mutation +obstruction. The scheduler reserves successful candidate footprints, constructs +one composite consequence, and emits one Tick receipt entry per Action. The successor state remains private until the complete Tick transaction is durable. Construction failure discards it. WAL failure restores the From 0b76eaaa0cabbb7848558700a8290830a7717e9a Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:24:40 -0700 Subject: [PATCH 61/64] Fix: drain after Action admission progress --- CHANGELOG.md | 2 + crates/warp-core/src/trusted_runtime_host.rs | 23 ++++- .../executable_operation_pipeline_tests.rs | 85 +++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c66aaa54..34295bcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,8 @@ complete limit with independent, non-conflicting node targets. Admission applies that ceiling independently to each runnable head, so Actions retained for dormant or faulted heads cannot consume another head's Tick capacity. + `run_until_idle` continues after a no-Step pass that advances bounded Action + admission, so an obstructed prefix cannot hide later admissible work. When more than that ceiling is pending, runtime admission selects the bounded set by canonical ingress identity rather than submission identity. `TrustedRuntimeHost::into_parts` now returns an opaque diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 4554fd38..d17a049e 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -380,6 +380,13 @@ pub struct TrustedRuntimeHostRunReport { pub committed_steps: usize, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct EchoOperationActionSchedulerPostureV1 { + pending: usize, + admitted: usize, + admission_obstructions: usize, +} + /// One Echo causal-anchor admission observed in committed control history. /// /// The enclosed fact and receipt are the semantic evidence. WAL coordinates on @@ -1868,6 +1875,14 @@ impl TrustedRuntimeHost { } } + fn echo_operation_action_scheduler_posture_v1(&self) -> EchoOperationActionSchedulerPostureV1 { + EchoOperationActionSchedulerPostureV1 { + pending: self.pending_echo_operation_actions.len(), + admitted: self.admitted_echo_operation_actions.len(), + admission_obstructions: self.echo_operation_action_admission_obstructions.len(), + } + } + fn requeue_obstructed_echo_operation_actions_v1(&mut self) { self.pending_echo_operation_actions.extend( self.echo_operation_action_admission_obstructions @@ -2133,7 +2148,8 @@ impl TrustedRuntimeHost { Ok(records) } - /// Runs scheduler-owned passes until an idle pass occurs. + /// Runs scheduler-owned passes until one pass commits no steps and makes + /// no executable-Action admission progress. /// /// # Errors /// @@ -2156,9 +2172,12 @@ impl TrustedRuntimeHost { max_scheduler_passes, }); } + let action_posture_before = self.echo_operation_action_scheduler_posture_v1(); let steps = self.tick_once()?; + let action_admission_progressed = + self.echo_operation_action_scheduler_posture_v1() != action_posture_before; report.scheduler_passes += 1; - if steps.is_empty() { + if steps.is_empty() && !action_admission_progressed { return Ok(report); } report.committed_steps += steps.len(); diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index eb2689bd..7975e3f0 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3620,6 +3620,91 @@ fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { assert_eq!(recovered_node, node); } +#[test] +fn run_until_idle_continues_after_admission_only_progress() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the bounded-admission fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let valid_invocation = + action_invocation(&host, &installed, head_key, node, b"before", b"valid-after"); + let valid_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + valid_invocation, + ) + .expect("the valid invocation becomes an Action"); + let valid_ingress_id = valid_envelope.ingress_id(); + + let unavailable_package_bytes = operation_package_at( + "echo.fixture.BoundedUnavailable.v1", + make_type_id("operation-fixture-node"), + make_type_id("operation-fixture-atom"), + ) + .to_canonical_bytes() + .expect("the unavailable package is canonical"); + let unavailable_package_id = + warp_core::echo_operation_package_id_v1(&unavailable_package_bytes); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis()) + .expect("Echo resolves the current exact parent basis"); + let mut unavailable_envelopes = Vec::new(); + for ordinal in 0..4096_u32 { + let unavailable_invocation = + EchoOperationInvocationV1::anchored_node_attachment_compare_and_set( + unavailable_package_id, + "echo.fixture.BoundedUnavailable.v1", + evaluation_basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + node, + warp_core::echo_operation_atom_value_digest_v1( + make_type_id("operation-fixture-atom"), + b"before", + ), + format!("unavailable-{ordinal}").into_bytes(), + ) + .to_canonical_bytes() + .expect("the unavailable invocation is canonical"); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + unavailable_invocation, + ) + .expect("the unavailable invocation becomes an Action"); + if envelope.ingress_id() < valid_ingress_id { + unavailable_envelopes.push(envelope); + if unavailable_envelopes.len() == ACTION_BATCH_CANDIDATE_LIMIT_V1 { + break; + } + } + } + assert_eq!( + unavailable_envelopes.len(), + ACTION_BATCH_CANDIDATE_LIMIT_V1, + "the deterministic fixture must fill the first bounded admission prefix" + ); + for envelope in unavailable_envelopes { + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("each unavailable Action is durable"); + } + let valid_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(valid_envelope) + .expect("the later valid Action is durable") + .submission_id; + + let report = host + .run_until_idle(4) + .expect("admission-only progress keeps the bounded drain alive"); + assert_eq!(report.committed_steps, 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + #[test] fn action_outcome_attribution_ignores_application_controlled_scope_collisions() { let (mut host, head_key, node) = fixture_host(); From 77c7dc8822dce90f45d5717bd2b08be108b3cce9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:33:29 -0700 Subject: [PATCH 62/64] Fix: restore whole-inbox partition admission --- CHANGELOG.md | 2 ++ crates/warp-core/src/head_inbox.rs | 45 +++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34295bcc..e319a182 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ global-Tick parity. The scheduler-round coordinate advances once per pass, so every head switches categories even when several heads share one worldline, preventing caller-controlled ingress hashes from starving either category. + Homogeneous unbounded admission moves the complete inbox map instead of + rebuilding and removing its entries one by one. Positional receipt attribution is exclusive to executable Actions; ordinary correlations without an exact scope match fail closed. Scheduler selection admits at most 64 executable Actions per Tick, leaving diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index 597753b3..f82701a8 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -680,6 +680,8 @@ pub struct HeadInbox { head_key: WriterHeadKey, pending: BTreeMap, policy: InboxPolicy, + #[cfg(test)] + partitioned_filtered_removals_for_test: usize, } impl Default for HeadInbox { @@ -691,6 +693,8 @@ impl Default for HeadInbox { }, pending: BTreeMap::new(), policy: InboxPolicy::AcceptAll, + #[cfg(test)] + partitioned_filtered_removals_for_test: 0, } } } @@ -703,6 +707,8 @@ impl HeadInbox { head_key, pending: BTreeMap::new(), policy, + #[cfg(test)] + partitioned_filtered_removals_for_test: 0, } } @@ -843,6 +849,9 @@ impl HeadInbox { if limit == 0 { return Vec::new(); } + if !(has_partition && has_other) && limit >= self.pending.len() { + return std::mem::take(&mut self.pending).into_values().collect(); + } let mut selected_ids = Vec::new(); for (ingress_id, envelope) in &self.pending { @@ -860,7 +869,13 @@ impl HeadInbox { } selected_ids .into_iter() - .filter_map(|ingress_id| self.pending.remove(&ingress_id)) + .filter_map(|ingress_id| { + #[cfg(test)] + { + self.partitioned_filtered_removals_for_test += 1; + } + self.pending.remove(&ingress_id) + }) .collect() } @@ -1030,6 +1045,34 @@ mod tests { assert!(inbox.is_empty()); } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn homogeneous_unbounded_partitioned_admission_moves_the_whole_inbox() { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::AcceptAll, + ); + let partition_kind = test_kind(); + for payload in [b"partition-a", b"partition-b", b"partition-c"] { + assert_eq!( + inbox.ingest(make_envelope(partition_kind, payload)), + InboxIngestResult::Accepted + ); + } + + let admitted = inbox.admit_partitioned(partition_kind, usize::MAX, crate::GlobalTick::ZERO); + + assert_eq!(admitted.len(), 3); + assert!(inbox.is_empty()); + assert_eq!( + inbox.partitioned_filtered_removals_for_test, 0, + "homogeneous unbounded admission must move the map without per-key removals" + ); + } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] #[test] fn scheduler_round_alternates_each_head_despite_even_worldline_progress() { From 606b5e0d3f11c943492601ea689abe28396449ac Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:35:40 -0700 Subject: [PATCH 63/64] Fix: scope partition admission test instrumentation --- crates/warp-core/src/head_inbox.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index f82701a8..233ac05f 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -680,7 +680,7 @@ pub struct HeadInbox { head_key: WriterHeadKey, pending: BTreeMap, policy: InboxPolicy, - #[cfg(test)] + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] partitioned_filtered_removals_for_test: usize, } @@ -693,7 +693,7 @@ impl Default for HeadInbox { }, pending: BTreeMap::new(), policy: InboxPolicy::AcceptAll, - #[cfg(test)] + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] partitioned_filtered_removals_for_test: 0, } } @@ -707,7 +707,7 @@ impl HeadInbox { head_key, pending: BTreeMap::new(), policy, - #[cfg(test)] + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] partitioned_filtered_removals_for_test: 0, } } @@ -870,7 +870,7 @@ impl HeadInbox { selected_ids .into_iter() .filter_map(|ingress_id| { - #[cfg(test)] + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] { self.partitioned_filtered_removals_for_test += 1; } From aed42bc274398e1772f340ffb3043bda265dbab2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Fri, 24 Jul 2026 21:39:47 -0700 Subject: [PATCH 64/64] Fix: borrow Action outcomes during WAL staging --- CHANGELOG.md | 3 ++- crates/warp-core/src/trusted_runtime_host.rs | 26 +++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e319a182..5de35b67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,7 +63,8 @@ Recovered Action outcomes resolve installed packages through one package-ID index instead of scanning the complete installation set per Action. Test instrumentation now counts the actual package-index and installation-order - lookups rather than asserting constant zero-value proxies. + lookups rather than asserting constant zero-value proxies. Tick WAL staging + borrows its read-only outcome index instead of deep-cloning every outcome. The v1 scheduler Action-candidate ceiling is exported as `ACTION_BATCH_CANDIDATE_LIMIT_V1`; its acceptance witness now proves the complete limit with independent, non-conflicting node targets. Admission diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index d17a049e..85480b56 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -2066,8 +2066,10 @@ impl TrustedRuntimeHost { tick_wal_records.push((correlation, decision, state_delta, state_delta_digest)); } } - let action_outcomes_by_submission = - action_outcomes.iter().cloned().collect::>(); + let action_outcomes_by_submission = action_outcomes + .iter() + .map(|(submission_id, outcome)| (*submission_id, outcome)) + .collect::>(); let mut tick_wal_groups = BTreeMap::new(); for (correlation, decision, state_delta, state_delta_digest) in tick_wal_records { let key = ( @@ -2963,7 +2965,7 @@ impl TrustedRuntimeWal { fn record_tick_receipt_batch( &mut self, correlations: &[(ReceiptCorrelationRecord, WalTickDecision)], - action_outcomes_by_submission: &BTreeMap, + action_outcomes_by_submission: &BTreeMap, state_delta: &WalRuntimeStateDeltaRecord, state_delta_digest: Hash, ) -> Result { @@ -6181,6 +6183,24 @@ mod tests { }; use bytes::Bytes; + type BorrowedActionOutcomeBatchWriter = + for<'a> fn( + &mut TrustedRuntimeWal, + &[(ReceiptCorrelationRecord, WalTickDecision)], + &BTreeMap, + &WalRuntimeStateDeltaRecord, + Hash, + ) -> Result; + + #[test] + fn tick_receipt_batch_accepts_borrowed_action_outcomes() { + let writer: BorrowedActionOutcomeBatchWriter = TrustedRuntimeWal::record_tick_receipt_batch; + assert!(std::ptr::fn_addr_eq( + writer, + TrustedRuntimeWal::record_tick_receipt_batch as BorrowedActionOutcomeBatchWriter + )); + } + fn test_head_key() -> WriterHeadKey { WriterHeadKey { worldline_id: WorldlineId::from_bytes([9; 32]),