diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8754f4b..57dbe01f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,10 @@ jobs: run: | cargo clippy -p warp-core --features host_test --test causal_fact_publication_tests -- -D warnings -D missing_docs cargo clippy -p warp-core --features host_test --test optic_invocation_admission_tests -- -D warnings -D missing_docs + - name: cargo clippy (scheduler-owned executable Actions) + run: | + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests -- -D warnings -D missing_docs + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs clippy-det-fixed: name: Clippy (det_fixed) @@ -301,6 +305,10 @@ jobs: run: cargo test -p warp-core --features native_rule_bootstrap,host_test --test installed_contract_intent_pipeline_tests - name: cargo test (warp-core provider proposal admission) run: cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime --test provider_contract_admission_tests + - name: cargo test (scheduler-owned executable Actions) + run: | + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests - name: cargo test --doc (warp-core) run: cargo test -p warp-core --doc - name: PRNG golden regression (warp-math) diff --git a/.github/workflows/macos-local.yml b/.github/workflows/macos-local.yml index 5a1d3810..e1d52b78 100644 --- a/.github/workflows/macos-local.yml +++ b/.github/workflows/macos-local.yml @@ -51,6 +51,10 @@ jobs: run: | cargo clippy -p warp-core --features host_test --test causal_fact_publication_tests -- -D warnings -D missing_docs cargo clippy -p warp-core --features host_test --test optic_invocation_admission_tests -- -D warnings -D missing_docs + - name: cargo clippy (scheduler-owned executable Actions) + run: | + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests -- -D warnings -D missing_docs + cargo clippy -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests -- -D warnings -D missing_docs - name: cargo test (workspace) run: cargo test --workspace - name: cargo test (warp-core host_test admission fixtures) @@ -59,3 +63,7 @@ jobs: cargo test -p warp-core --features host_test --test optic_invocation_admission_tests - name: cargo test (warp-core installed contract intent pipeline) run: cargo test -p warp-core --features native_rule_bootstrap,host_test --test installed_contract_intent_pipeline_tests + - name: cargo test (scheduler-owned executable Actions) + run: | + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime --test executable_operation_pipeline_tests + cargo test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests diff --git a/CHANGELOG.md b/CHANGELOG.md index c7c4714b..5de35b67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,92 @@ ### Added +- Executable-operation application writes now enter Echo as canonical, + WAL-acknowledged Actions and are evaluated only by the scheduler while + constructing a Tick (ADR 0025). Accepted pre-Tick Actions recover as pending + work. Runtime-owned admission uses a bounded pending index and cache; + unavailable packages are quarantined without poisoning unrelated work. + Durable-acceptance lookup and newly committed receipt correlations are + indexed directly, so scheduler Ticks do not replay or diff retained history. + A WAL-enabled app surface rejects executable Actions submitted outside the + durable acknowledgement boundary before witnessed intake can mutate. + Mixed executable and provider/native backlogs alternate by durable parent + global-Tick parity. The scheduler-round coordinate advances once per pass, so + every head switches categories even when several heads share one worldline, + preventing caller-controlled ingress hashes from starving either category. + Homogeneous unbounded admission moves the complete inbox map instead of + rebuilding and removing its entries one by one. + Positional receipt attribution is exclusive to executable Actions; ordinary + correlations without an exact scope match fail closed. + Scheduler selection admits at most 64 executable Actions per Tick, leaving + excess work pending, and meters footprint comparisons, blocker evidence, and + aggregate operations during composition. Two independent Actions can share + one exact parent coordinate and contribute to one composite Tick while + retaining candidate-specific application-basis propositions and per-Action + typed outcomes. Footprint conflicts name earlier applied members; evaluator + and composition-budget obstructions contribute no operations. Every + noncommitted typed outcome carries its deciding writer head, worldline and + global Tick coordinates, commit and Tick-receipt identities, and canonical + member index. Tick construction pins that index to the corresponding receipt + entry with an executable alignment invariant. + One scheduler WAL transaction retains exactly one batched Tick decision + record, then each Action's receipt correlation and typed outcome in canonical + order, followed by exactly one replayable state delta. Recovery rejects every + second transaction claiming the same state-transition coordinate, including + a byte-identical delta, so a Tick cannot be reconstructed from split WAL + fragments. The decided Tick is durable before state, frontier, receipt, or + outcome publication. Recovery + and same-host retry both preserve an accepted Action when Tick-WAL + persistence fails; runtime rollback also rolls back the corresponding + admission cache so the Action re-enters scheduler admission. Fresh-host + recovery validates every outcome against its exact envelope, admission, + invocation, installed operation, causal coordinate, evaluation basis, + reconstructed preparation and actual footprint, Tick entry, composite + consequence, exact reconstructed aggregate patch membership, and state root. + Typed obstruction records retain their invocation-admission policy and + budget ceiling; recovery reproduces bounded evaluation and the complete + scheduler composition before accepting an obstruction kind. For a + cross-worldline basis obstruction, recovery reconstructs the submitted basis + on its named worldline but resolves the deciding transition on the target + head's worldline. + Recovery records one monotonic installation ordinal per package and one + installation-count boundary per Action, avoiding per-outcome package-set + snapshots while preserving exact installation-before-Tick validation. + Activation caches each reconstructed causal state by worldline coordinate, + so Actions sharing one scheduler basis do not replay that history repeatedly. + Recovered Action outcomes resolve installed packages through one package-ID + index instead of scanning the complete installation set per Action. Test + instrumentation now counts the actual package-index and installation-order + lookups rather than asserting constant zero-value proxies. Tick WAL staging + borrows its read-only outcome index instead of deep-cloning every outcome. + The v1 scheduler Action-candidate ceiling is exported as + `ACTION_BATCH_CANDIDATE_LIMIT_V1`; its acceptance witness now proves the + complete limit with independent, non-conflicting node targets. Admission + applies that ceiling independently to each runnable head, so Actions retained + for dormant or faulted heads cannot consume another head's Tick capacity. + `run_until_idle` continues after a no-Step pass that advances bounded Action + admission, so an obstructed prefix cannot hide later admissible work. + When more than that ceiling is pending, runtime admission selects the bounded + set by canonical ingress identity rather than submission identity. + `TrustedRuntimeHost::into_parts` now returns an opaque + `TrustedRuntimeHostParts` value consumed by `from_parts`, preserving WAL, + authority, policy, pending admission, and every typed Action outcome across + host decomposition and reconstruction. + A composite receipt cannot validate outside its complete Action-batch + context. Legacy operation recovery-index roots remain byte-compatible when + no Action outcome exists. Direct operation + prepare/commit remains a documentation-hidden public + `TrustedRuntimeHost` compatibility/test seam, absent from + `TrustedRuntimeApp`; removal is tracked by issue #689. - `TrustedRuntimeHost` now has the first hook-free executable-operation runtime slice. A runtime owner can admit exact canonical `ExecutableOperationPackageV1` bytes under a separate package policy, install their data-only `EchoOperationProgramV1`, independently admit an exact-basis invocation under caller authority and delegated budget, evaluate privately, and either commit one parent-visible patch or return typed noncommit evidence. - Only committed operation consequences enter the operation-tick WAL. The initial - generic program performs an anchored typed-node alpha-attachment + On that transitional direct seam, only committed operation consequences + enter the operation-tick WAL. The initial generic program performs an + anchored typed-node alpha-attachment compare-and-set; it contains no application matcher, executor, footprint callback, or prebuilt mutation plan. Package, installation, invocation, evaluation, actual-footprint, budget, patch, result, basis, and terminal @@ -28,8 +106,7 @@ callbacks. A program digest alone cannot install, invoke, or authorize an operation. This slice does not yet include Edict compiler emission, a structurally separate target verifier, Jedit's rope - lawpack, `ReplaceRange`, scheduler batch composition, or an independently - implemented semantic oracle. + lawpack, `ReplaceRange`, or an independently implemented semantic oracle. - The executable-operation corridor now has a separate `AnchoredNodeAttachmentCreateIfAbsent` program (ADR 0024). The original compare-and-set program remains update-only with its canonical program, diff --git a/README.md b/README.md index 4cad6add..49ab07be 100644 --- a/README.md +++ b/README.md @@ -175,25 +175,26 @@ host-constructed `InstalledContractPackage` values. Separately, Echo can now admit and install an exact canonical `ExecutableOperationPackageV1`, admit an exact-basis invocation under explicit authority and delegated budget, evaluate its data-only -`EchoOperationProgramV1` privately, and either commit one patch or return typed -noncommit evidence. Only committed operation consequences enter the -operation-tick WAL. The initial generic program performs an anchored typed-node -alpha-attachment compare-and-set. Its receipt binds the admitted package, -operation, subordinate program, invocation, complete evaluation basis, -authority, declared and actual footprints, budgets, patch, result, and terminal -outcome. Runtime-control installation and committed execution-kernel records -permit callback-free fresh-host recovery. Program bytes explicitly bind the -interpreter and intrinsic profiles, while the parent patch and singleton tick -evidence bind the admitted installation. A program digest cannot confer -operation identity, invocability, or authority. +`EchoOperationProgramV1` privately during scheduler-owned Tick construction, +and either commit its member consequence or return typed noncommit evidence. +Two independent, same-basis executable Actions can enter one atomic Tick. The +initial generic programs perform anchored typed-node alpha-attachment +compare-and-set and create-if-absent operations. Their receipts bind the +admitted package, operation, subordinate program, invocation, complete +evaluation basis, authority, declared and actual footprints, budgets, patch, +result, composition, and terminal outcome. Runtime-control installation and +scheduler Tick records permit callback-free fresh-host recovery. Program bytes +explicitly bind the interpreter and intrinsic profiles, while Tick evidence +binds each admitted installation. A program digest cannot confer operation +identity, invocability, or authority. That generic runtime witness is not yet the Jim/Jedit vertical. No real Edict compiler output, Jedit rope lawpack, or `ReplaceRange` operation uses it, and it -does not yet claim structurally separate target verification, scheduler batch -composition, or independently implemented semantic conformance. It also -temporarily reuses `TrustedRuntimeHost`'s joint `native_rule_bootstrap` and -`trusted_runtime` feature gate. The program itself has no native hooks, but the -host surface must be decoupled from the legacy bootstrap feature before a +does not yet claim structurally separate target verification, cross-category +scheduler composition, or independently implemented semantic conformance. It +also temporarily reuses `TrustedRuntimeHost`'s joint `native_rule_bootstrap` +and `trusted_runtime` feature gate. The program itself has no native hooks, but +the host surface must be decoupled from the legacy bootstrap feature before a product can remove that compatibility feature. The following sequence is the existing Wesley bootstrap fixture: @@ -248,17 +249,23 @@ The current executable-operation runtime slice is: canonical ExecutableOperationPackageV1 bytes -> Echo-owned package and invocation admission -> installed data-only EchoOperationProgramV1 --> bounded private Echo evaluation --> exact-basis singleton commit --> typed receipt, WAL retention, and callback-free recovery +-> canonical Action submission retained before acknowledgement +-> runtime-owned admission into the ordinary head inbox +-> scheduler selection at one exact basis +-> bounded private Echo evaluation during Tick construction +-> one composite Tick consequence with typed per-Action outcomes +-> decided Tick WAL retention before state, frontier, and Receipt publication +-> callback-free pending-Action and decided-Tick recovery ``` The first two paths are callback-shaped compatibility infrastructure. The third proves Echo-owned execution of admitted data-only meaning through separate update-only compare-and-set and single-node create-if-absent program -profiles. No real Edict compiler output, Jedit operation, or Graft -multi-record mutation uses them yet. The next convergence crossing must bind a -real application-owned Edict operation and lawpack to the executable-operation +profiles. Independent Actions for one head can share one scheduler-owned Tick; +conflicting or obstructed Actions retain typed outcomes without hidden +mutation. No real Edict compiler output, Jedit operation, or Graft operation +uses this route yet. The next convergence crossing must bind a real +application-owned Edict operation and lawpack to the executable-operation package without reintroducing a native implementation. ## Contracts And Boundaries diff --git a/crates/warp-core/src/causal_wal.rs b/crates/warp-core/src/causal_wal.rs index b6399c3c..e8b57349 100644 --- a/crates/warp-core/src/causal_wal.rs +++ b/crates/warp-core/src/causal_wal.rs @@ -73,6 +73,7 @@ use crate::{CausalTickReceiptRef, CAUSAL_TICK_RECEIPT_REF_LEN}; const WAL_FRAME_DOMAIN: &[u8] = b"echo:causal_wal:frame:v1\0"; const WAL_PAYLOAD_DOMAIN: &[u8] = b"echo:causal_wal:payload:v1\0"; const WAL_TICK_RECEIPT_MAGIC_V2: &[u8; 8] = b"ETICK002"; +const WAL_TICK_RECEIPT_BATCH_MAGIC_V3: &[u8; 8] = b"ETICK003"; const WAL_RECEIPT_CORRELATION_MAGIC_V2: &[u8; 8] = b"ERCOR002"; const LEGACY_TICK_RECEIPT_PAYLOAD_LEN: usize = 3 * core::mem::size_of::() + 1; const LEGACY_RECEIPT_CORRELATION_PREFIX_LEN: usize = 3 * core::mem::size_of::(); @@ -446,6 +447,9 @@ pub enum WalRecordKind { ExecutableOperationExecutionRecorded, /// Echo execution kernel retained the outcome's replayable state delta. ExecutableOperationStateDeltaRecorded, + /// Trusted scheduler retained one typed executable-operation Action + /// outcome inside a scheduler-owned Tick. + ExecutableOperationActionOutcomeRecorded, } impl WalRecordKind { @@ -479,6 +483,9 @@ impl WalRecordKind { Self::ExecutableOperationPackageInstalled => "ExecutableOperationPackageInstalled", Self::ExecutableOperationExecutionRecorded => "ExecutableOperationExecutionRecorded", Self::ExecutableOperationStateDeltaRecorded => "ExecutableOperationStateDeltaRecorded", + Self::ExecutableOperationActionOutcomeRecorded => { + "ExecutableOperationActionOutcomeRecorded" + } } } @@ -494,6 +501,7 @@ impl WalRecordKind { | Self::TickReceiptRecorded | Self::RuntimeStateDeltaRecorded | Self::ReceiptCorrelationRecorded + | Self::ExecutableOperationActionOutcomeRecorded | Self::ReadingEnvelopeRetained | Self::RetainedMaterialRefRecorded | Self::MaterializationIntentRecorded @@ -558,6 +566,7 @@ impl WalRecordKind { Self::ExecutableOperationPackageInstalled => 25, Self::ExecutableOperationExecutionRecorded => 26, Self::ExecutableOperationStateDeltaRecorded => 27, + Self::ExecutableOperationActionOutcomeRecorded => 28, } } @@ -590,6 +599,7 @@ impl WalRecordKind { 25 => Ok(Self::ExecutableOperationPackageInstalled), 26 => Ok(Self::ExecutableOperationExecutionRecorded), 27 => Ok(Self::ExecutableOperationStateDeltaRecorded), + 28 => Ok(Self::ExecutableOperationActionOutcomeRecorded), _ => Err(WalDecodeError::UnknownEnumCode { enum_name: "WalRecordKind", code, @@ -2436,7 +2446,7 @@ impl WalTickDecision { /// Returns `true` when this decision is a lawful rejection, not a fault. #[must_use] pub const fn is_lawful_rejection(self) -> bool { - matches!(self, Self::RejectedFootprintConflict) + matches!(self, Self::RejectedFootprintConflict | Self::Obstructed) } } @@ -2493,6 +2503,108 @@ impl TickReceiptRecord { } } +#[derive(Clone, Debug, PartialEq, Eq)] +struct TickReceiptBatchRecord { + members: Vec, +} + +impl TickReceiptBatchRecord { + fn new(members: Vec) -> Result { + let Some(first) = members.first() else { + return Err(WalBuildError::TickBatchReceiptShapeMismatch); + }; + if members + .iter() + .any(|member| !same_scheduler_tick(&first.receipt_ref, &member.receipt_ref)) + { + return Err(WalBuildError::TickBatchReceiptShapeMismatch); + } + Ok(Self { members }) + } + + fn to_payload_bytes(&self) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(WAL_TICK_RECEIPT_BATCH_MAGIC_V3); + out.extend_from_slice(&len_u64(self.members.len()).to_le_bytes()); + for member in &self.members { + out.extend_from_slice(&member.receipt_ref.to_canonical_bytes()); + out.push(member.decision.code()); + } + out + } + + fn from_payload_bytes(bytes: &[u8]) -> Result { + let mut cursor = WalPayloadCursor::new(bytes); + if cursor.read_exact(WAL_TICK_RECEIPT_BATCH_MAGIC_V3.len())? + != WAL_TICK_RECEIPT_BATCH_MAGIC_V3 + { + return Err(WalDecodeError::InvalidRecordMagic { + record_kind: "tick-receipt-batch", + }); + } + let member_count = cursor.read_u64()?; + let member_width = CAUSAL_TICK_RECEIPT_REF_LEN + 1; + let maximum_encoded_members = u64::try_from( + bytes + .len() + .saturating_sub(WAL_TICK_RECEIPT_BATCH_MAGIC_V3.len() + 8) + / member_width, + ) + .unwrap_or(u64::MAX); + if member_count == 0 || member_count > maximum_encoded_members { + return Err(WalDecodeError::InvalidEmbeddedFrame); + } + let member_count = + usize::try_from(member_count).map_err(|_| WalDecodeError::UnexpectedEof)?; + let mut members = Vec::with_capacity(member_count); + for _ in 0..member_count { + let receipt_ref = CausalTickReceiptRef::from_canonical_bytes( + cursor + .read_exact(CAUSAL_TICK_RECEIPT_REF_LEN)? + .try_into() + .map_err(|_| WalDecodeError::UnexpectedEof)?, + ); + let decision = WalTickDecision::from_code(cursor.read_u8()?)?; + members.push(TickReceiptRecord { + receipt_ref, + decision, + }); + } + cursor.finish()?; + let first = &members[0].receipt_ref; + if members + .iter() + .any(|member| !same_scheduler_tick(first, &member.receipt_ref)) + { + return Err(WalDecodeError::InvalidEmbeddedFrame); + } + Ok(Self { members }) + } +} + +pub(crate) fn decode_tick_receipt_records( + bytes: &[u8], +) -> Result, WalDecodeError> { + if bytes.starts_with(WAL_TICK_RECEIPT_BATCH_MAGIC_V3) { + TickReceiptBatchRecord::from_payload_bytes(bytes).map(|batch| batch.members) + } else { + TickReceiptRecord::from_payload_bytes(bytes).map(|receipt| vec![receipt]) + } +} + +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +pub(crate) fn tick_receipt_payload_is_batch(bytes: &[u8]) -> bool { + bytes.starts_with(WAL_TICK_RECEIPT_BATCH_MAGIC_V3) +} + +fn same_scheduler_tick(left: &CausalTickReceiptRef, right: &CausalTickReceiptRef) -> bool { + left.worldline_id == right.worldline_id + && left.worldline_tick_after == right.worldline_tick_after + && left.commit_global_tick == right.commit_global_tick + && left.commit_hash == right.commit_hash + && left.receipt_content_digest == right.receipt_content_digest +} + /// WAL receipt correlation record payload. #[derive(Clone, Debug, PartialEq, Eq)] pub struct WalReceiptCorrelationRecord { @@ -7771,18 +7883,76 @@ pub fn build_tick_transaction( /// Builds a scheduler-owned tick transaction with replayable state-delta material. pub fn build_replayable_tick_transaction( - mut builder: WalTransactionBuilder, + builder: WalTransactionBuilder, receipt: TickReceiptRecord, correlation: WalReceiptCorrelationRecord, retained_state_delta_bytes: Vec, affected_frontiers: Vec, ) -> Result { + build_replayable_tick_batch_transaction( + builder, + vec![(receipt, correlation, None)], + retained_state_delta_bytes, + affected_frontiers, + ) +} + +/// Builds one scheduler-owned Tick transaction with one or more per-Action +/// receipt correlations and exactly one replayable state consequence. +pub(crate) fn build_replayable_tick_batch_transaction( + mut builder: WalTransactionBuilder, + receipts: Vec<( + TickReceiptRecord, + WalReceiptCorrelationRecord, + Option>, + )>, + retained_state_delta_bytes: Vec, + affected_frontiers: Vec, +) -> Result { + let action_outcome_count = receipts + .iter() + .filter(|(_, _, action_outcome)| action_outcome.is_some()) + .count(); + if action_outcome_count != 0 && action_outcome_count != receipts.len() { + return Err(WalBuildError::TickBatchActionOutcomeShapeMismatch); + } let state_delta = WalRuntimeStateDeltaRecord::from_payload_bytes(&retained_state_delta_bytes) .map_err(|_| WalBuildError::RuntimeStateDeltaInvalid)?; - if state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest { + if receipts.is_empty() + || receipts.iter().any(|(receipt, correlation, _)| { + state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest + || receipt.receipt_ref != correlation.receipt_ref + }) + { return Err(WalBuildError::RuntimeStateDeltaReceiptMismatch); } - push_tick_receipt_records(&mut builder, receipt, &correlation)?; + if action_outcome_count == receipts.len() { + let receipt_batch = + TickReceiptBatchRecord::new(receipts.iter().map(|(receipt, _, _)| *receipt).collect())?; + builder.push_record( + WalRecordKind::TickReceiptRecorded, + receipt_batch.to_payload_bytes(), + )?; + for (receipt, correlation, action_outcome) in receipts { + if receipt.receipt_ref != correlation.receipt_ref { + return Err(WalBuildError::ReceiptCorrelationMismatch); + } + builder.push_record( + WalRecordKind::ReceiptCorrelationRecorded, + correlation.to_payload_bytes(), + )?; + let action_outcome = + action_outcome.ok_or(WalBuildError::TickBatchActionOutcomeShapeMismatch)?; + builder.push_record( + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + action_outcome, + )?; + } + } else { + for (receipt, correlation, _) in receipts { + push_tick_receipt_records(&mut builder, receipt, &correlation)?; + } + } builder.push_record( WalRecordKind::RuntimeStateDeltaRecorded, retained_state_delta_bytes, @@ -8401,9 +8571,9 @@ pub fn recover_submission_index( index.insert_acceptance_record(record)?; } WalRecordKind::TickReceiptRecorded => { - let receipt = - TickReceiptRecord::from_payload_bytes(&frame.payload.canonical_bytes)?; - index.apply_tick_receipt_record(receipt)?; + for receipt in decode_tick_receipt_records(&frame.payload.canonical_bytes)? { + index.apply_tick_receipt_record(receipt)?; + } } _ => {} } @@ -8421,9 +8591,9 @@ pub fn recover_receipt_index( for frame in &transaction.frames { match frame.header.record_kind { WalRecordKind::TickReceiptRecorded => { - let receipt = - TickReceiptRecord::from_payload_bytes(&frame.payload.canonical_bytes)?; - index.apply_tick_receipt_record(receipt)?; + for receipt in decode_tick_receipt_records(&frame.payload.canonical_bytes)? { + index.apply_tick_receipt_record(receipt)?; + } } WalRecordKind::ReceiptCorrelationRecorded => { let correlation = WalReceiptCorrelationRecord::from_payload_bytes( @@ -8852,6 +9022,12 @@ pub enum WalBuildError { /// Replayable runtime state delta names a different receipt commitment. #[error("WAL replayable runtime state delta does not match the tick receipt")] RuntimeStateDeltaReceiptMismatch, + /// A scheduler Tick batch mixes Action and non-Action receipt members. + #[error("WAL scheduler Tick batch has inconsistent Action outcome framing")] + TickBatchActionOutcomeShapeMismatch, + /// A scheduler Tick batch contains no receipt members or spans Ticks. + #[error("WAL scheduler Tick batch receipt shape is inconsistent")] + TickBatchReceiptShapeMismatch, /// Validation failed. #[error(transparent)] Validation(#[from] WalValidationError), diff --git a/crates/warp-core/src/causal_wal_tests.rs b/crates/warp-core/src/causal_wal_tests.rs index 70806e49..78332947 100644 --- a/crates/warp-core/src/causal_wal_tests.rs +++ b/crates/warp-core/src/causal_wal_tests.rs @@ -14,9 +14,9 @@ use crate::causal_wal::{ apply_committed_transaction, audit_wal_release_readiness, build_causal_anchor_admission_transaction, build_checkpoint_publication_transaction, build_materialization_outbox_transaction, build_recovery_certificate, - build_retained_reading_transaction, build_submission_acceptance_transaction, - build_submission_acceptance_with_material_transaction, build_tick_transaction, - build_topology_intent_transaction, canonical_segment_relative_path, + build_replayable_tick_batch_transaction, build_retained_reading_transaction, + build_submission_acceptance_transaction, build_submission_acceptance_with_material_transaction, + build_tick_transaction, build_topology_intent_transaction, canonical_segment_relative_path, causal_anchor_frontier_digest_from_evidence, causal_anchor_genesis_frontier_digest, causal_history_genesis_frontier_digest, doctor_in_memory_store, evaluate_checkpoint_publication, lint_wal_schema_terms, logical_causal_history_frontier_digest, @@ -3184,6 +3184,37 @@ fn tick_transaction_rejects_mismatched_receipt_correlation() { assert_eq!(error, WalBuildError::ReceiptCorrelationMismatch); } +#[test] +fn tick_batch_rejects_mixed_action_outcome_framing() { + let error = must_err( + build_replayable_tick_batch_transaction( + builder( + transaction_id("tx:tick:mixed-action-outcomes"), + Lsn::from_raw(0), + WalAppendAuthority::TrustedScheduler, + WalTransactionKind::SchedulerTick, + ), + vec![ + ( + receipt_record("action", WalTickDecision::Applied), + correlation_record("action"), + Some(b"retained-action-outcome".to_vec()), + ), + ( + receipt_record("legacy", WalTickDecision::Applied), + correlation_record("legacy"), + None, + ), + ], + b"state-delta-is-not-decoded-before-shape-validation".to_vec(), + Vec::new(), + ), + "one scheduler Tick must not mix Action and non-Action receipt members", + ); + + assert_eq!(error, WalBuildError::TickBatchActionOutcomeShapeMismatch); +} + #[test] fn receipt_correlation_decode_rejects_noncanonical_parent_sets() { let parent_a = causal_receipt_ref("parent-a"); @@ -3716,6 +3747,7 @@ fn lawful_rejection_recovers_without_fault_posture() { .copied() ) .is_lawful_rejection()); + assert!(WalTickDecision::Obstructed.is_lawful_rejection()); } #[test] diff --git a/crates/warp-core/src/contract_obstruction.rs b/crates/warp-core/src/contract_obstruction.rs index cda0806f..75fa6d99 100644 --- a/crates/warp-core/src/contract_obstruction.rs +++ b/crates/warp-core/src/contract_obstruction.rs @@ -228,6 +228,12 @@ impl ContractObstruction { submission_id: *submission_id, }, ), + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationCommit(_) + | RuntimeError::EchoOperationActionAdmissionMissing(_) + | RuntimeError::EchoOperationActionRequiresRuntimeWalAck => { + Self::runtime_fault(ContractObstructionSubject::Unspecified) + } RuntimeError::SchedulerFaultGenerationOverflow | RuntimeError::DuplicateWorldline(_) | RuntimeError::DuplicateHead(_) diff --git a/crates/warp-core/src/coordinator.rs b/crates/warp-core/src/coordinator.rs index 411ff112..27054cc4 100644 --- a/crates/warp-core/src/coordinator.rs +++ b/crates/warp-core/src/coordinator.rs @@ -11,6 +11,13 @@ use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; use thiserror::Error; use crate::clock::{GlobalTick, WorldlineTick}; +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +use crate::echo_operation::{ + AdmittedEchoOperationInvocationV1, EchoOperationActionOutcomeV1, + EchoOperationApplicationBasisV1, EchoOperationCommitErrorV1, + EchoOperationEvaluationAuthorityV1, EchoOperationEvaluationBasisV1, + SchedulerEchoOperationCandidateV1, +}; use crate::engine_impl::{CommitOutcome, Engine, EngineError}; use crate::head::{ HeadEligibility, PlaybackHeadRegistry, RunnableWriterSet, WriterHead, WriterHeadKey, @@ -40,6 +47,49 @@ use crate::CausalTickReceiptRef; #[cfg(feature = "native_rule_bootstrap")] const INSTALLED_CONTRACT_EINT_INTENT_KIND_LABEL: &str = "echo.intent/eint-v1"; +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +type SchedulerOperationOutcomeV1 = (Hash, EchoOperationActionOutcomeV1); +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +type SchedulerOperationOutcomesV1 = Vec; +#[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] +type SchedulerOperationOutcomeV1 = (); +#[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] +type SchedulerOperationOutcomesV1 = Vec; + +#[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] +pub(crate) fn resolve_echo_operation_evaluation_basis_v1( + runtime: &WorldlineRuntime, + provenance: &ProvenanceService, + writer_head: WriterHeadKey, + application_basis: EchoOperationApplicationBasisV1, +) -> Result { + if runtime.heads().get(&writer_head).is_none() { + return Err(RuntimeError::UnknownHead(writer_head)); + } + let frontier = runtime + .worldlines() + .get(&writer_head.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(writer_head.worldline_id))?; + let state_root = frontier.state().state_root(); + let (commit_global_tick, commit_id) = match provenance.tip_ref(writer_head.worldline_id)? { + Some(tip) => { + let entry = provenance.entry(tip.worldline_id, tip.worldline_tick)?; + (Some(entry.commit_global_tick), entry.expected.commit_hash) + } + None => ( + None, + crate::echo_operation::genesis_commit_id(writer_head, state_root), + ), + }; + Ok(EchoOperationEvaluationBasisV1::new( + writer_head, + frontier.frontier_tick(), + commit_global_tick, + state_root, + commit_id, + application_basis, + )) +} // ============================================================================= // Runtime Errors and Ingress Disposition @@ -88,9 +138,26 @@ pub enum RuntimeError { /// Ordinary submission attempted to claim the contract-inverse target role. #[error("contract inverse target parent requires contract inverse admission")] ContractInverseTargetRequiresContractAdmission, + /// A WAL-enabled host received an executable-operation Action through the + /// non-durable app submission method. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[error( + "executable-operation Actions on a WAL-enabled host require submit_intent_with_runtime_wal_ack" + )] + EchoOperationActionRequiresRuntimeWalAck, /// A commit against a worldline frontier failed. #[error(transparent)] Engine(#[from] EngineError), + /// A scheduler-owned executable-operation Action batch could not be + /// constructed. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[error(transparent)] + EchoOperationCommit(#[from] EchoOperationCommitErrorV1), + /// The scheduler selected a reserved executable Action without the + /// runtime-owned admission token that authorizes evaluation. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[error("executable-operation Action admission is unavailable for ingress {0:?}")] + EchoOperationActionAdmissionMissing(Hash), /// Provenance append or lookup failed during a runtime step. #[error(transparent)] Provenance(#[from] HistoryError), @@ -1081,6 +1148,11 @@ pub struct WorldlineRuntime { next_submission_generation: IngressSubmissionGeneration, /// Witnessed submission records keyed by content-addressed submission id. witnessed_submissions: BTreeMap, + /// Undecided witnessed submission ids. + /// + /// This is a derived, recovery-rebuilt scheduler index. It keeps pending + /// admission proportional to outstanding work instead of retained history. + pending_witnessed_submission_ids: BTreeSet, /// Canonical envelope material for witnessed submissions. witnessed_submission_envelopes: BTreeMap, /// Deterministic lookup from resolved semantic target and ingress id to @@ -1105,6 +1177,8 @@ pub struct WorldlineRuntime { /// admitted by that commit. receipt_correlations_by_current_basis: BTreeMap<(WorldlineId, WorldlineTick, Hash), BTreeSet>, + #[cfg(any(test, feature = "host_test"))] + receipt_correlation_full_scan_count: std::cell::Cell, /// Scheduler fault evidence keyed by content-addressed fault id. scheduler_faults: BTreeMap, /// Active scoped scheduler faults by writer head. @@ -1116,6 +1190,12 @@ pub struct WorldlineRuntime { next_scheduler_fault_generation: SchedulerFaultGeneration, /// Registry of live speculative strands attached to the runtime. strands: StrandRegistry, + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + fail_next_echo_operation_action_tick_construction: bool, } #[derive(Clone, Debug)] @@ -1137,6 +1217,7 @@ struct ReceiptCorrelationRollbackEntry { previous_receipt_ref_ticketed_ingress: Option, current_basis: (WorldlineId, WorldlineTick, Hash), previous_current_basis_ticketed_ingresses: Option>, + previous_pending_submission: bool, } #[derive(Clone, Debug, Default)] @@ -1259,6 +1340,13 @@ impl WorldlineRuntime { .remove(&entry.current_basis); } } + if entry.previous_pending_submission { + self.pending_witnessed_submission_ids + .insert(entry.submission_id); + } else { + self.pending_witnessed_submission_ids + .remove(&entry.submission_id); + } } } @@ -1268,6 +1356,24 @@ impl WorldlineRuntime { &self.worldlines } + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + pub(crate) fn inject_echo_operation_action_tick_construction_failure_for_test(&mut self) { + self.fail_next_echo_operation_action_tick_construction = true; + } + + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + fn take_echo_operation_action_tick_construction_failure_for_test(&mut self) -> bool { + std::mem::take(&mut self.fail_next_echo_operation_action_tick_construction) + } + /// Returns the registered writer heads. #[must_use] pub fn heads(&self) -> &PlaybackHeadRegistry { @@ -1295,6 +1401,33 @@ impl WorldlineRuntime { self.witnessed_submissions.values() } + /// Iterates only undecided witnessed submissions in deterministic id order. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn pending_witnessed_submissions( + &self, + ) -> impl Iterator { + self.pending_witnessed_submission_ids + .iter() + .filter_map(|submission_id| self.witnessed_submissions.get(submission_id)) + } + + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn witnessed_submission_id_for_target( + &self, + head_key: WriterHeadKey, + ingress_id: Hash, + ) -> Option { + self.submission_by_target + .get(&(head_key, ingress_id)) + .copied() + } + + /// Returns the number of witnessed submissions without a receipt decision. + #[must_use] + pub fn pending_witnessed_submission_count(&self) -> usize { + self.pending_witnessed_submission_ids.len() + } + /// Returns witnessed submissions in deterministic replay order. /// /// Replay order follows Echo-owned submission generation, then submission id @@ -1452,9 +1585,29 @@ impl WorldlineRuntime { /// Iterates receipt correlations in deterministic ticketed-ingress id order. pub fn receipt_correlations(&self) -> impl Iterator { + #[cfg(any(test, feature = "host_test"))] + self.receipt_correlation_full_scan_count.set( + self.receipt_correlation_full_scan_count + .get() + .saturating_add(1), + ); self.receipt_correlations_by_ticketed_ingress.values() } + /// Resets test instrumentation for full receipt-correlation scans. + #[cfg(any(test, feature = "host_test"))] + pub fn reset_receipt_correlation_full_scan_count_for_test(&self) { + self.receipt_correlation_full_scan_count.set(0); + } + + /// Returns the number of full receipt-correlation scans since the last + /// instrumentation reset. + #[cfg(any(test, feature = "host_test"))] + #[must_use] + pub fn receipt_correlation_full_scan_count_for_test(&self) -> usize { + self.receipt_correlation_full_scan_count.get() + } + /// Returns the number of scheduler-owned receipt correlations. #[must_use] pub fn receipt_correlation_count(&self) -> usize { @@ -1634,6 +1787,40 @@ impl WorldlineRuntime { .iter() .enumerate() .filter(|(_idx, entry)| entry.scope.local_id == ingress_node); + let positional_candidate = || { + let current_basis = receipt_correlation_current_basis(correlation); + let ticketed_ingress_ids = self + .receipt_correlations_by_current_basis + .get(¤t_basis)?; + let mut same_tick = ticketed_ingress_ids + .iter() + .filter_map(|ticketed_ingress_id| { + self.receipt_correlations_by_ticketed_ingress + .get(ticketed_ingress_id) + }) + .collect::>(); + same_tick.sort_by_key(|candidate| candidate.ingress_id); + if same_tick.len() != receipt.entries().len() + || same_tick + .windows(2) + .any(|pair| pair[0].ingress_id >= pair[1].ingress_id) + { + return None; + } + let index = same_tick.iter().position(|candidate| { + candidate.ticketed_ingress_id == correlation.ticketed_ingress_id + })?; + receipt.entries().get(index).map(|entry| (index, entry)) + }; + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let is_echo_operation_action = self + .witnessed_submission_envelopes + .get(&correlation.submission_id) + .is_some_and(|envelope| { + crate::echo_operation::echo_operation_action_invocation_bytes_v1(envelope).is_some() + }); + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + let is_echo_operation_action = false; let candidate = if let Some(provider) = correlation .contract .as_ref() @@ -1649,11 +1836,18 @@ impl WorldlineRuntime { return no_match(); } candidate - } else { - let Some(candidate) = candidates.next() else { + } else if is_echo_operation_action { + let Some(candidate) = positional_candidate() else { return no_match(); }; candidate + } else if let Some(candidate) = candidates.next() { + if candidates.next().is_some() { + return no_match(); + } + candidate + } else { + return no_match(); }; let (idx, entry) = candidate; let Ok(receipt_entry_index) = u32::try_from(idx) else { @@ -2031,6 +2225,16 @@ impl WorldlineRuntime { self.next_submission_generation = next_submission_generation; self.submission_by_target .extend(staged_submission_by_target); + self.pending_witnessed_submission_ids.extend( + staged_witnessed_submissions + .keys() + .filter(|submission_id| { + !self + .receipt_correlation_by_submission + .contains_key(*submission_id) + }) + .copied(), + ); self.witnessed_submissions .extend(staged_witnessed_submissions); Ok(()) @@ -2207,6 +2411,7 @@ impl WorldlineRuntime { .entries() .iter() .any(|entry| entry.scope.local_id == ingress_node) + && crate::echo_operation::echo_operation_action_invocation_bytes_v1(envelope).is_none() { return Err(RuntimeError::ReceiptCorrelationReplayMismatch( persisted.causal_receipt_ref.identity_digest(), @@ -2320,6 +2525,8 @@ impl WorldlineRuntime { .entry(current_basis) .or_default() .insert(ticketed_ingress_id); + self.pending_witnessed_submission_ids + .remove(&persisted.submission_id); self.worldlines .frontier_mut(&persisted.head_key.worldline_id) .ok_or(RuntimeError::UnknownWorldline( @@ -2437,6 +2644,23 @@ impl WorldlineRuntime { self.ingest_ticketed_invocation_inner(submission_id, ticket.ticket_digest, envelope, None) } + /// Stages one runtime-admitted executable-operation Action into the normal + /// head inbox. + /// + /// The opaque admission digest is derived by the trusted runtime owner from + /// exact installed meaning and invocation-admission evidence. No contract + /// callback evidence is attached to this ingress category. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn ingest_echo_operation_action_v1( + &mut self, + _authority: &TicketedRuntimeIngressAuthority, + submission_id: Hash, + admission_digest: Hash, + envelope: IngressEnvelope, + ) -> Result { + self.ingest_ticketed_invocation_inner(submission_id, admission_digest, envelope, None) + } + fn ingest_ticketed_invocation_inner( &mut self, submission_id: Hash, @@ -2708,6 +2932,7 @@ impl WorldlineRuntime { .insert((head_key, ingress_id), submission_id); self.witnessed_submissions .insert(submission_id, record.clone()); + self.pending_witnessed_submission_ids.insert(submission_id); Ok(record) } @@ -2807,7 +3032,8 @@ impl WorldlineRuntime { admitted: &[IngressEnvelope], context: ReceiptCorrelationCommitContext, rollback: &mut ReceiptCorrelationRollback, - ) -> Result<(), RuntimeError> { + ) -> Result, RuntimeError> { + let mut committed = Vec::new(); for envelope in admitted { let ingress_id = envelope.ingress_id(); let Some(ticketed_ingress_id) = self @@ -2898,7 +3124,11 @@ impl WorldlineRuntime { .receipt_correlations_by_current_basis .get(¤t_basis) .cloned(), + previous_pending_submission: self + .pending_witnessed_submission_ids + .contains(&ticketed_ingress.submission_id), }); + committed.push(record.clone()); self.receipt_correlations_by_ticketed_ingress .insert(ticketed_ingress_id, record); self.receipt_correlation_by_submission @@ -2911,8 +3141,10 @@ impl WorldlineRuntime { .entry(current_basis) .or_default() .insert(ticketed_ingress_id); + self.pending_witnessed_submission_ids + .remove(&ticketed_ingress.submission_id); } - Ok(()) + Ok(committed) } fn resolve_target(&self, target: &IngressTarget) -> Result { @@ -3042,6 +3274,8 @@ fn scheduler_fault_scope_for_error( RuntimeError::Engine(_) | RuntimeError::FrontierTickOverflow(_) => { SchedulerFaultScope::Head(head_key) } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationCommit(_) => SchedulerFaultScope::Head(head_key), RuntimeError::Provenance(_) | RuntimeError::UnknownHead(_) | RuntimeError::UnknownWorldline(_) @@ -3074,6 +3308,9 @@ fn scheduler_fault_scope_for_error( | RuntimeError::TicketedIngressDuplicateRuntimeIngress { .. } => { SchedulerFaultScope::Runtime } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationActionAdmissionMissing(_) + | RuntimeError::EchoOperationActionRequiresRuntimeWalAck => SchedulerFaultScope::Runtime, } } @@ -3714,6 +3951,20 @@ fn scheduler_error_cause_digest(err: &RuntimeError) -> Hash { } } } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationCommit(error) => { + hasher.update(b"echo-operation-commit"); + hasher.update(error.to_string().as_bytes()); + } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationActionAdmissionMissing(ingress_id) => { + hasher.update(b"echo-operation-action-admission-missing"); + hasher.update(ingress_id); + } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + RuntimeError::EchoOperationActionRequiresRuntimeWalAck => { + hasher.update(b"echo-operation-action-requires-runtime-wal-ack"); + } RuntimeError::FrontierTickOverflow(worldline_id) => { hasher.update(b"frontier-tick-overflow"); hasher.update(worldline_id.as_bytes()); @@ -3917,12 +4168,65 @@ impl SchedulerCoordinator { provenance: &mut ProvenanceService, engine: &mut Engine, ) -> Result, RuntimeError> { + Self::super_tick_inner(runtime, provenance, engine, None) + .map(|(records, _operation_outcomes, _correlations)| records) + } + + /// Executes one scheduler pass with runtime-admitted executable-operation + /// Actions available to Tick construction. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn super_tick_with_echo_operation_actions_v1( + runtime: &mut WorldlineRuntime, + provenance: &mut ProvenanceService, + engine: &mut Engine, + operation_actions: &BTreeMap, + evaluation_authority: &EchoOperationEvaluationAuthorityV1, + ) -> Result< + ( + Vec, + SchedulerOperationOutcomesV1, + Vec, + ), + RuntimeError, + > { + Self::super_tick_inner( + runtime, + provenance, + engine, + Some((operation_actions, evaluation_authority)), + ) + } + + fn super_tick_inner( + runtime: &mut WorldlineRuntime, + provenance: &mut ProvenanceService, + engine: &mut Engine, + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + operation_actions: Option<( + &BTreeMap, + &EchoOperationEvaluationAuthorityV1, + )>, + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + _operation_actions: Option<()>, + ) -> Result< + ( + Vec, + SchedulerOperationOutcomesV1, + Vec, + ), + RuntimeError, + > { if let Some(fault_id) = runtime.runtime_fault { return Err(RuntimeError::SchedulerRuntimeFaultActive(fault_id)); } runtime.refresh_runnable(); let mut records = Vec::new(); + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let mut operation_outcomes = Vec::new(); + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + let operation_outcomes = Vec::new(); + let mut committed_correlations = Vec::new(); let keys: Vec = runtime.runnable.iter().copied().collect(); let next_global_tick = if let Some(next) = runtime.global_tick.checked_increment() { next @@ -3961,11 +4265,24 @@ impl SchedulerCoordinator { provenance.checkpoint_for(keys.iter().map(|key| key.worldline_id))?; for key in &keys { - let admitted = runtime + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let partition_parent_global_tick = runtime.global_tick; + let inbox = runtime .heads .inbox_mut(key) - .ok_or(RuntimeError::UnknownHead(*key))? - .admit(); + .ok_or(RuntimeError::UnknownHead(*key))?; + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let admitted = if operation_actions.is_some() { + inbox.admit_partitioned( + crate::echo_operation::echo_operation_action_intent_kind_v1(), + crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1, + partition_parent_global_tick, + ) + } else { + inbox.admit() + }; + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] + let admitted = inbox.admit(); if admitted.is_empty() { continue; @@ -3979,6 +4296,99 @@ impl SchedulerCoordinator { .frontier_tick(); let parents = provenance.tip_ref(key.worldline_id)?.into_iter().collect(); + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + // `HeadInbox::admit_partitioned` guarantees that one admitted + // batch contains either executable-operation Actions or + // non-Action work, never both. Inspecting the first member is + // therefore a batch classification, not a positional guess. + let executable_action_batch = admitted.first().is_some_and(|envelope| { + crate::echo_operation::echo_operation_action_invocation_bytes_v1(envelope) + .is_some() + }); + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + let (snapshot, patch, receipt) = if executable_action_batch { + #[cfg(any(test, feature = "host_test"))] + if runtime.take_echo_operation_action_tick_construction_failure_for_test() { + return Err(RuntimeError::EchoOperationCommit( + EchoOperationCommitErrorV1::InjectedTickConstructionFailure, + )); + } + let Some((admitted_actions, evaluation_authority)) = operation_actions else { + return Err(RuntimeError::EchoOperationActionAdmissionMissing( + admitted[0].ingress_id(), + )); + }; + let mut candidates = Vec::with_capacity(admitted.len()); + for envelope in &admitted { + let ingress_id = envelope.ingress_id(); + let submission_id = runtime + .witnessed_submission_id_for_target(*key, ingress_id) + .ok_or(RuntimeError::EchoOperationActionAdmissionMissing( + ingress_id, + ))?; + let admitted_action = admitted_actions.get(&submission_id).ok_or( + RuntimeError::EchoOperationActionAdmissionMissing(ingress_id), + )?; + let current_basis = resolve_echo_operation_evaluation_basis_v1( + runtime, + provenance, + *key, + admitted_action.evaluation_basis().application_basis(), + )?; + let state = runtime + .worldlines + .get(&key.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))? + .state(); + let preparation = crate::echo_operation::prepare_operation_v1( + engine + .installed_echo_operation_package_v1(admitted_action.package_id()), + admitted_action.clone(), + current_basis, + state, + engine.echo_operation_policy_id(), + evaluation_authority, + ); + candidates.push(SchedulerEchoOperationCandidateV1 { + submission_id, + ingress_id, + scope: admitted_action.scope(), + rule_id: admitted_action.installed_operation_id().as_hash(), + preparation, + }); + } + let material = { + let frontier = runtime + .worldlines + .frontier_mut(&key.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))?; + crate::echo_operation::commit_scheduler_action_batch_to_state_v1( + candidates, + *key, + frontier.state_mut(), + next_global_tick, + engine.echo_operation_policy_id(), + )? + }; + operation_outcomes.extend(material.outcomes); + (material.snapshot, material.patch, material.tick_receipt) + } else { + let CommitOutcome { + snapshot, + patch, + receipt, + } = { + let frontier = runtime + .worldlines + .frontier_mut(&key.worldline_id) + .ok_or(RuntimeError::UnknownWorldline(key.worldline_id))?; + engine + .commit_with_state(frontier.state_mut(), &admitted) + .map_err(RuntimeError::from)? + }; + (snapshot, patch, receipt) + }; + #[cfg(not(all(feature = "native_rule_bootstrap", feature = "trusted_runtime")))] let CommitOutcome { snapshot, patch, @@ -4046,7 +4456,7 @@ impl SchedulerCoordinator { .ok_or(RuntimeError::FrontierTickOverflow(key.worldline_id))?; (snapshot.state_root, worldline_tick_after) }; - runtime.record_receipt_correlations( + committed_correlations.extend(runtime.record_receipt_correlations( &admitted, ReceiptCorrelationCommitContext { head_key: *key, @@ -4056,7 +4466,7 @@ impl SchedulerCoordinator { commit_hash: snapshot.hash, }, &mut receipt_correlation_rollback, - )?; + )?); Ok(StepRecord { head_key: *key, @@ -4100,7 +4510,7 @@ impl SchedulerCoordinator { } runtime.global_tick = next_global_tick; - Ok(records) + Ok((records, operation_outcomes, committed_correlations)) } /// Returns the canonical ordering of runnable heads without mutating state. @@ -6265,6 +6675,60 @@ mod tests { assert!(obstruction.contract.is_some()); } + #[test] + fn ordinary_outcome_never_uses_positional_action_attribution() { + let (mut runtime, mut engine, worldline_id) = toy_contract_runtime(); + let head_key = runtime + .resolve_target(&IngressTarget::DefaultWriter { worldline_id }) + .expect("the toy runtime has a default writer"); + let envelope = IngressEnvelope::local_intent( + IngressTarget::DefaultWriter { worldline_id }, + make_intent_kind("echo.intent/eint-v1"), + toy_increment_intent(TOY_INCREMENT_VARS), + ); + let mut provenance = mirrored_provenance(&runtime); + let mut correlation = commit_ticketed_envelope( + &mut runtime, + &mut provenance, + &mut engine, + head_key, + envelope, + hash(91), + ); + let receipt = &runtime + .worldlines() + .get(&worldline_id) + .expect("the toy worldline remains registered") + .state() + .tick_history() + .last() + .expect("the toy intent commits one Tick") + .1; + assert_eq!(receipt.entries().len(), 1); + correlation.ingress_id = (92_u8..=u8::MAX) + .map(hash) + .find(|candidate| { + receipt + .entries() + .iter() + .all(|entry| entry.scope.local_id != NodeId(*candidate)) + }) + .expect("the fixture provides an ingress id outside the receipt scopes"); + runtime + .receipt_correlations_by_ticketed_ingress + .insert(correlation.ticketed_ingress_id, correlation.clone()); + + assert!(matches!( + runtime.observe_intent_outcome(&correlation.submission_id), + IntentOutcomeObservation::Decided { + decision: IntentOutcomeDecision::NoMatchingReceiptEntry { + tick_receipt_digest, + }, + .. + } if tick_receipt_digest == correlation.tick_receipt_digest + )); + } + #[test] fn submission_order_does_not_define_scheduler_order() { let worldline_id = wl(1); diff --git a/crates/warp-core/src/echo_operation.rs b/crates/warp-core/src/echo_operation.rs index 9d3d69bd..73df1f60 100644 --- a/crates/warp-core/src/echo_operation.rs +++ b/crates/warp-core/src/echo_operation.rs @@ -32,9 +32,10 @@ use crate::{ attachment::{AtomPayload, AttachmentKey, AttachmentValue}, clock::{GlobalTick, WorldlineTick}, footprint::{Footprint, WarpScopedPortKey}, - head::WriterHeadKey, + head::{HeadId, WriterHeadKey}, + head_inbox::{make_intent_kind, IngressEnvelope, IngressPayload, IngressTarget, IntentKind}, ident::{EdgeKey, Hash, NodeKey, TypeId}, - receipt::{TickReceipt, TickReceiptDisposition, TickReceiptEntry}, + receipt::{TickReceipt, TickReceiptDisposition, TickReceiptEntry, TickReceiptRejection}, record::NodeRecord, snapshot::{compute_commit_hash_v2, Snapshot}, tick_patch::{SlotId, TickCommitStatus, TickPatchError, WarpOp, WarpTickPatchV1}, @@ -78,6 +79,19 @@ const INTRINSIC_PROFILE: &str = "echo.operation-attachment-algebra/v1"; const PACKAGE_ID_DOMAIN: &[u8] = b"echo:operation-package:v1\0"; const PROGRAM_ID_DOMAIN: &[u8] = b"echo:operation-program:v1\0"; const INVOCATION_ID_DOMAIN: &[u8] = b"echo:operation-invocation:v1\0"; +const ACTION_INTENT_KIND: &str = "echo.executable-operation-action/v1"; +const ACTION_BATCH_RULE_PACK_DOMAIN: &[u8] = b"echo:operation-action-batch-rule-pack:v1\0"; +const ACTION_BATCH_PLAN_DOMAIN: &[u8] = b"echo:operation-action-batch-plan:v1\0"; +const ACTION_BATCH_REWRITES_DOMAIN: &[u8] = b"echo:operation-action-batch-rewrites:v1\0"; +const ACTION_BATCH_COMPOSITION_DOMAIN: &[u8] = b"echo:operation-action-batch-composition:v1\0"; +const ACTION_OUTCOME_RECORD_MAGIC: &[u8; 8] = b"EOACT003"; +/// Maximum number of executable-operation Action candidates selected for one +/// scheduler-owned Tick under the v1 bounded-composition profile. +pub const ACTION_BATCH_CANDIDATE_LIMIT_V1: usize = 64; +const ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1: usize = + ACTION_BATCH_CANDIDATE_LIMIT_V1 * (ACTION_BATCH_CANDIDATE_LIMIT_V1 - 1) / 2; +const ACTION_BATCH_BLOCKER_EVIDENCE_LIMIT_V1: usize = ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1; +const ACTION_BATCH_OPERATION_LIMIT_V1: usize = ACTION_BATCH_CANDIDATE_LIMIT_V1 * 2; const INVOCATION_BYTES_DIGEST_DOMAIN: &[u8] = b"echo:operation-invocation-bytes:v1\0"; const BASIS_ID_DOMAIN: &[u8] = b"echo:operation-evaluation-basis:v1\0"; const PACKAGE_ADMISSION_ID_DOMAIN: &[u8] = b"echo:operation-package-admission:v1\0"; @@ -270,6 +284,49 @@ pub fn echo_operation_package_id_v1(bytes: &[u8]) -> EchoOperationPackageIdV1 { EchoOperationPackageIdV1(domain_hash(PACKAGE_ID_DOMAIN, bytes)) } +/// Returns the reserved stable kind for Echo-interpreted executable-operation +/// Actions. +#[must_use] +pub fn echo_operation_action_intent_kind_v1() -> IntentKind { + make_intent_kind(ACTION_INTENT_KIND) +} + +/// Wraps one canonical executable-operation invocation in the ordinary Echo +/// Action envelope. +/// +/// The helper decodes and reproduces the invocation before constructing +/// ingress, so non-canonical or unsupported bytes never acquire the reserved +/// Action kind. +pub fn echo_operation_action_envelope_v1( + target: IngressTarget, + canonical_invocation_bytes: Vec, +) -> Result { + let invocation = EchoOperationInvocationV1::from_canonical_bytes(&canonical_invocation_bytes)?; + if invocation.to_canonical_bytes()? != canonical_invocation_bytes { + return Err(artifact_error( + EchoOperationArtifactErrorKindV1::NonCanonical, + "executable-operation Action must carry exact canonical invocation bytes", + )); + } + Ok(IngressEnvelope::local_intent( + target, + echo_operation_action_intent_kind_v1(), + canonical_invocation_bytes, + )) +} + +pub(crate) fn echo_operation_action_invocation_bytes_v1( + envelope: &IngressEnvelope, +) -> Option<&[u8]> { + match envelope.payload() { + IngressPayload::LocalIntent { + intent_kind, + intent_bytes, + } if *intent_kind == echo_operation_action_intent_kind_v1() => Some(intent_bytes), + IngressPayload::LocalIntent { .. } => None, + } +} + /// Computes the digest of one typed attachment atom. #[must_use] pub fn echo_operation_atom_value_digest_v1(type_id: TypeId, bytes: &[u8]) -> Hash { @@ -2235,6 +2292,18 @@ impl AdmittedEchoOperationInvocationV1 { pub(crate) const fn evaluation_basis(&self) -> EchoOperationEvaluationBasisV1 { self.invocation.evaluation_basis } + + pub(crate) const fn scope(&self) -> NodeKey { + self.invocation.node + } + + pub(crate) const fn installed_operation_id(&self) -> InstalledEchoOperationIdV1 { + self.installed_operation_id + } + + pub(crate) const fn admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { + self.admission_id + } } pub(crate) fn admit_invocation_v1( @@ -2245,6 +2314,85 @@ pub(crate) fn admit_invocation_v1( current_state: &WorldlineState, evaluation_authority: EchoOperationEvaluationAuthorityV1, ) -> Result { + let (invocation, installed) = + admit_invocation_static_v1(installed, policy, canonical_invocation_bytes)?; + if invocation.evaluation_basis != current_basis { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation does not name the current exact parent basis", + )); + } + if invocation + .evaluation_basis + .application_basis + .schema_identity + != installed.application_basis_schema_identity + { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation application-basis schema differs from installed package", + )); + } + let current_application_basis = + current_application_basis(installed, &invocation, current_state)?; + if invocation.evaluation_basis.application_basis != current_application_basis { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation application basis differs from Echo's current graph proposition", + )); + } + Ok(build_admitted_invocation_v1( + invocation, + installed, + policy, + canonical_invocation_bytes, + evaluation_authority, + )) +} + +/// Admits the static installed meaning and runtime-owned authority for one +/// scheduler-bound executable-operation Action. +/// +/// Exact basis freshness is deliberately deferred to private evaluation during +/// Tick construction. That lets an Action which was durably accepted at an +/// earlier basis reach the evaluator and receive a typed `BasisChanged` +/// obstruction instead of failing the host admission loop. +pub(crate) fn admit_action_invocation_v1( + installed: Option<&InstalledEchoOperationV1>, + policy: EchoOperationInvocationAdmissionPolicyV1, + canonical_invocation_bytes: &[u8], + evaluation_authority: EchoOperationEvaluationAuthorityV1, +) -> Result { + let (invocation, installed) = + admit_invocation_static_v1(installed, policy, canonical_invocation_bytes)?; + if invocation + .evaluation_basis + .application_basis + .schema_identity + != installed.application_basis_schema_identity + { + return Err(invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, + "invocation application-basis schema differs from installed package", + )); + } + Ok(build_admitted_invocation_v1( + invocation, + installed, + policy, + canonical_invocation_bytes, + evaluation_authority, + )) +} + +fn admit_invocation_static_v1<'a>( + installed: Option<&'a InstalledEchoOperationV1>, + policy: EchoOperationInvocationAdmissionPolicyV1, + canonical_invocation_bytes: &[u8], +) -> Result< + (EchoOperationInvocationV1, &'a InstalledEchoOperationV1), + EchoOperationInvocationAdmissionErrorV1, +> { let invocation = EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) .map_err(|error| { invocation_admission_error( @@ -2298,38 +2446,24 @@ pub(crate) fn admit_invocation_v1( "delegated budget is below the program minimum or exceeds an admitted ceiling", )); } - if invocation.evaluation_basis != current_basis { - return Err(invocation_admission_error( - EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, - "invocation does not name the current exact parent basis", - )); - } - if invocation - .evaluation_basis - .application_basis - .schema_identity - != installed.application_basis_schema_identity - { - return Err(invocation_admission_error( - EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, - "invocation application-basis schema differs from installed package", - )); - } - let current_application_basis = - current_application_basis(installed, &invocation, current_state)?; - if invocation.evaluation_basis.application_basis != current_application_basis { - return Err(invocation_admission_error( - EchoOperationInvocationAdmissionErrorKindV1::BasisMismatch, - "invocation application basis differs from Echo's current graph proposition", - )); - } + Ok((invocation, installed)) +} + +fn build_admitted_invocation_v1( + invocation: EchoOperationInvocationV1, + installed: &InstalledEchoOperationV1, + policy: EchoOperationInvocationAdmissionPolicyV1, + canonical_invocation_bytes: &[u8], + evaluation_authority: EchoOperationEvaluationAuthorityV1, +) -> AdmittedEchoOperationInvocationV1 { let invocation_id = EchoOperationInvocationIdV1(domain_hash( INVOCATION_ID_DOMAIN, canonical_invocation_bytes, )); let admission_policy_id = policy.identity(); let installed_operation_id = installed.installed_operation_id; - Ok(AdmittedEchoOperationInvocationV1 { + let evaluation_basis_id = invocation.evaluation_basis.identity(); + AdmittedEchoOperationInvocationV1 { invocation, invocation_id, canonical_invocation_bytes: canonical_invocation_bytes.to_vec(), @@ -2338,12 +2472,12 @@ pub(crate) fn admit_invocation_v1( installed_operation_id, invocation_id, admission_policy_id, - current_basis.identity(), + evaluation_basis_id, ), installed_operation_id, evaluation_authority, admission_policy: policy, - }) + } } fn current_application_basis( @@ -2413,6 +2547,134 @@ fn current_application_basis( } } +pub(crate) fn action_application_basis_matches_state_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + state: &WorldlineState, +) -> Result { + let invocation = EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) + .map_err(|error| { + invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::MalformedInvocation, + error.to_string(), + ) + })?; + Ok(current_application_basis(installed, &invocation, state)? + == invocation.evaluation_basis.application_basis) +} + +pub(crate) fn action_admission_evidence_matches_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + maximum_budget: EchoOperationBudgetV1, + expected_policy_id: Hash, + expected_admission_id: EchoOperationInvocationAdmissionIdV1, +) -> bool { + let Ok(invocation) = + EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) + else { + return false; + }; + let policy = EchoOperationInvocationAdmissionPolicyV1::new( + installed.authority_profile_identity, + invocation.authority_grant_identity, + maximum_budget, + ); + let policy_id = policy.identity(); + policy_id == expected_policy_id + && invocation_admission_id( + installed.installed_operation_id, + EchoOperationInvocationIdV1(domain_hash( + INVOCATION_ID_DOMAIN, + canonical_invocation_bytes, + )), + policy_id, + invocation.evaluation_basis.identity(), + ) == expected_admission_id +} + +pub(crate) fn action_preparation_identity_matches_v1( + private_evaluation_id: EchoOperationPrivateEvaluationIdV1, + prepared_patch_digest: Hash, + prepared_result_id: EchoOperationResultIdV1, + expected_preparation_id: PreparedEchoOperationIdV1, +) -> bool { + preparation_id( + private_evaluation_id, + prepared_patch_digest, + prepared_result_id, + ) == expected_preparation_id +} + +pub(crate) fn reconstruct_action_evaluation_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + maximum_budget: EchoOperationBudgetV1, + expected_policy_id: Hash, + expected_admission_id: EchoOperationInvocationAdmissionIdV1, + state: &WorldlineState, + policy_id: u32, +) -> Option { + if !action_admission_evidence_matches_v1( + installed, + canonical_invocation_bytes, + maximum_budget, + expected_policy_id, + expected_admission_id, + ) { + return None; + } + let invocation = + EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes).ok()?; + let policy = EchoOperationInvocationAdmissionPolicyV1::new( + installed.authority_profile_identity, + invocation.authority_grant_identity, + maximum_budget, + ); + let authority = EchoOperationEvaluationAuthorityV1::new(); + let admitted = admit_action_invocation_v1( + Some(installed), + policy, + canonical_invocation_bytes, + authority.clone(), + ) + .ok()?; + if admitted.admission_id() != expected_admission_id { + return None; + } + Some(prepare_operation_v1( + Some(installed), + admitted, + invocation.evaluation_basis, + state, + policy_id, + &authority, + )) +} + +pub(crate) fn reconstruct_action_preparation_v1( + installed: &InstalledEchoOperationV1, + canonical_invocation_bytes: &[u8], + maximum_budget: EchoOperationBudgetV1, + expected_policy_id: Hash, + expected_admission_id: EchoOperationInvocationAdmissionIdV1, + state: &WorldlineState, + policy_id: u32, +) -> Option> { + match reconstruct_action_evaluation_v1( + installed, + canonical_invocation_bytes, + maximum_budget, + expected_policy_id, + expected_admission_id, + state, + policy_id, + )? { + EchoOperationPreparationV1::Prepared(prepared) => Some(prepared), + EchoOperationPreparationV1::Obstructed(_) => None, + } +} + pub(crate) fn decode_invocation_route_v1( canonical_invocation_bytes: &[u8], ) -> Result< @@ -2429,6 +2691,39 @@ pub(crate) fn decode_invocation_route_v1( Ok((invocation.package_id, invocation.evaluation_basis)) } +pub(crate) struct EchoOperationActionInvocationEvidenceV1 { + pub package_id: EchoOperationPackageIdV1, + pub scope: NodeKey, + pub evaluation_basis: EchoOperationEvaluationBasisV1, + pub invocation_id: EchoOperationInvocationIdV1, + pub invocation_bytes_digest: Hash, +} + +pub(crate) fn inspect_action_invocation_v1( + canonical_invocation_bytes: &[u8], +) -> Result { + let invocation = EchoOperationInvocationV1::from_canonical_bytes(canonical_invocation_bytes) + .map_err(|error| { + invocation_admission_error( + EchoOperationInvocationAdmissionErrorKindV1::MalformedInvocation, + error.to_string(), + ) + })?; + Ok(EchoOperationActionInvocationEvidenceV1 { + package_id: invocation.package_id, + scope: invocation.node, + evaluation_basis: invocation.evaluation_basis, + invocation_id: EchoOperationInvocationIdV1(domain_hash( + INVOCATION_ID_DOMAIN, + canonical_invocation_bytes, + )), + invocation_bytes_digest: domain_hash( + INVOCATION_BYTES_DIGEST_DOMAIN, + canonical_invocation_bytes, + ), + }) +} + pub(crate) fn invocation_runtime_error( detail: impl Into, ) -> EchoOperationInvocationAdmissionErrorV1 { @@ -2445,9 +2740,14 @@ pub(crate) fn runtime_basis_obstruction( kind: EchoOperationObstructionKindV1::BasisChanged, package_id: admitted.invocation.package_id, installed_operation_id: admitted.installed_operation_id, - invocation_admission_id: admitted.admission_id, + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: admitted.admission_policy_id, + maximum_budget: admitted.admission_policy.maximum_delegated_budget, + admission_id: admitted.admission_id, + }), invocation_id: admitted.invocation_id, evaluation_basis_id: admitted.invocation.evaluation_basis.identity(), + decision_coordinate: None, }) } @@ -2482,48 +2782,678 @@ pub enum EchoOperationObstructionKindV1 { ReplacementTooLarge, } -/// One typed obstruction. Obstruction never carries a parent-visible patch. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct EchoOperationObstructionV1 { - kind: EchoOperationObstructionKindV1, - package_id: EchoOperationPackageIdV1, - installed_operation_id: InstalledEchoOperationIdV1, - invocation_admission_id: EchoOperationInvocationAdmissionIdV1, - invocation_id: EchoOperationInvocationIdV1, - evaluation_basis_id: EchoOperationEvaluationBasisIdV1, +/// Retained runtime policy evidence needed to reproduce one obstruction. +#[derive(Clone, Debug, PartialEq, Eq)] +struct EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: Hash, + maximum_budget: EchoOperationBudgetV1, + admission_id: EchoOperationInvocationAdmissionIdV1, +} + +/// Exact scheduler-owned Tick coordinate which decided one noncommitted Action +/// outcome. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EchoOperationActionDecisionCoordinateV1 { + writer_head: WriterHeadKey, + worldline_tick_after: WorldlineTick, + commit_global_tick: GlobalTick, + commit_id: Hash, + tick_receipt_digest: Hash, + member_index: u32, +} + +impl EchoOperationActionDecisionCoordinateV1 { + /// Returns the writer head whose scheduler Tick decided the Action. + #[must_use] + pub const fn writer_head(self) -> WriterHeadKey { + self.writer_head + } + + /// Returns the worldline coordinate after the deciding Tick. + #[must_use] + pub const fn worldline_tick_after(self) -> WorldlineTick { + self.worldline_tick_after + } + + /// Returns the runtime-global coordinate of the deciding Tick. + #[must_use] + pub const fn commit_global_tick(self) -> GlobalTick { + self.commit_global_tick + } + + /// Returns the deciding Tick's commit identity. + #[must_use] + pub const fn commit_id(self) -> Hash { + self.commit_id + } + + /// Returns the deciding Tick receipt's digest. + #[must_use] + pub const fn tick_receipt_digest(self) -> Hash { + self.tick_receipt_digest + } + + /// Returns this Action's canonical member index in the deciding Tick. + #[must_use] + pub const fn member_index(self) -> u32 { + self.member_index + } +} + +/// One typed obstruction. Obstruction never carries a parent-visible patch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EchoOperationObstructionV1 { + kind: EchoOperationObstructionKindV1, + package_id: EchoOperationPackageIdV1, + installed_operation_id: InstalledEchoOperationIdV1, + invocation_admission: Box, + invocation_id: EchoOperationInvocationIdV1, + evaluation_basis_id: EchoOperationEvaluationBasisIdV1, + decision_coordinate: Option>, +} + +impl EchoOperationObstructionV1 { + /// Returns the stable obstruction category. + #[must_use] + pub const fn kind(&self) -> EchoOperationObstructionKindV1 { + self.kind + } + + /// Returns the installed package whose evaluation was obstructed. + #[must_use] + pub const fn package_id(&self) -> EchoOperationPackageIdV1 { + self.package_id + } + + /// Returns the exact installed operation against which evaluation was attempted. + #[must_use] + pub const fn installed_operation_id(&self) -> InstalledEchoOperationIdV1 { + self.installed_operation_id + } + + /// Returns the exact canonical invocation refused by evaluation. + #[must_use] + pub const fn invocation_id(&self) -> EchoOperationInvocationIdV1 { + self.invocation_id + } + + /// Returns the exact private-evaluation basis refused by evaluation. + #[must_use] + pub const fn evaluation_basis_id(&self) -> EchoOperationEvaluationBasisIdV1 { + self.evaluation_basis_id + } + + /// Returns the deciding Tick coordinate once this obstruction becomes a + /// terminal Action outcome. + #[must_use] + pub const fn decision_coordinate(&self) -> Option { + match &self.decision_coordinate { + Some(coordinate) => Some(**coordinate), + None => None, + } + } + + /// Returns the exact runtime admission that authorized this evaluation attempt. + #[must_use] + pub const fn invocation_admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { + self.invocation_admission.admission_id + } + + pub(crate) const fn invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission.policy_id + } + + pub(crate) const fn invocation_admission_maximum_budget(&self) -> EchoOperationBudgetV1 { + self.invocation_admission.maximum_budget + } + + pub(crate) fn has_same_predecision_evidence(&self, other: &Self) -> bool { + self.kind == other.kind + && self.package_id == other.package_id + && self.installed_operation_id == other.installed_operation_id + && self.invocation_admission == other.invocation_admission + && self.invocation_id == other.invocation_id + && self.evaluation_basis_id == other.evaluation_basis_id + } + + /// Returns the identity of this typed no-parent-patch obstruction. + #[must_use] + pub fn identity(&self) -> EchoOperationObstructionIdV1 { + let mut hasher = Hasher::new(); + hasher.update(OBSTRUCTION_ID_DOMAIN); + hasher.update(&self.package_id.as_hash()); + hasher.update(&self.installed_operation_id.as_hash()); + hasher.update(&self.invocation_admission.admission_id.as_hash()); + hasher.update(&self.invocation_id.as_hash()); + hasher.update(&self.evaluation_basis_id.as_hash()); + hasher.update(&[obstruction_kind_code(self.kind)]); + EchoOperationObstructionIdV1(hasher.finalize().into()) + } +} + +/// Complete retained evidence for a privately prepared Action rejected during +/// scheduler composition. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EchoOperationFootprintConflictV1 { + /// Exact installed package which admitted the rejected preparation. + pub(crate) package_id: EchoOperationPackageIdV1, + /// Exact installed operation whose prepared consequence conflicted. + pub(crate) installed_operation_id: InstalledEchoOperationIdV1, + /// Runtime policy identity used to admit the rejected invocation. + pub(crate) invocation_admission_policy_id: Hash, + /// Runtime policy ceiling used to admit the rejected invocation. + pub(crate) invocation_admission_maximum_budget: EchoOperationBudgetV1, + /// Exact invocation admission consumed by private evaluation. + pub(crate) invocation_admission_id: EchoOperationInvocationAdmissionIdV1, + /// Exact canonical invocation whose prepared consequence conflicted. + pub(crate) invocation_id: EchoOperationInvocationIdV1, + /// Exact evaluation basis consumed by private evaluation. + pub(crate) evaluation_basis_id: EchoOperationEvaluationBasisIdV1, + /// Exact private-evaluation identity produced before composition. + pub(crate) private_evaluation_id: EchoOperationPrivateEvaluationIdV1, + /// Exact prepared patch identity produced before composition. + pub(crate) prepared_patch_digest: Hash, + /// Exact prepared result identity produced before composition. + pub(crate) prepared_result_id: EchoOperationResultIdV1, + /// Exact actual footprint observed during private evaluation. + pub(crate) actual_footprint_digest: Hash, + /// Identity of the privately prepared candidate that was not applied. + pub(crate) preparation_id: PreparedEchoOperationIdV1, + /// Canonical indices of earlier applied Tick members that blocked it. + pub(crate) blocked_by: Vec, + /// Exact scheduler Tick and member position which decided the conflict. + pub(crate) decision_coordinate: Option>, +} + +impl EchoOperationFootprintConflictV1 { + /// Returns the exact package which admitted the rejected preparation. + #[must_use] + pub const fn package_id(&self) -> EchoOperationPackageIdV1 { + self.package_id + } + + /// Returns the installed operation whose prepared consequence conflicted. + #[must_use] + pub const fn installed_operation_id(&self) -> InstalledEchoOperationIdV1 { + self.installed_operation_id + } + + /// Returns the runtime policy identity used to admit the invocation. + #[must_use] + pub const fn invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission_policy_id + } + + /// Returns the runtime policy ceiling used to admit the invocation. + #[must_use] + pub const fn invocation_admission_maximum_budget(&self) -> EchoOperationBudgetV1 { + self.invocation_admission_maximum_budget + } + + /// Returns the invocation-admission evidence consumed by evaluation. + #[must_use] + pub const fn invocation_admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { + self.invocation_admission_id + } + + /// Returns the exact canonical invocation which was privately evaluated. + #[must_use] + pub const fn invocation_id(&self) -> EchoOperationInvocationIdV1 { + self.invocation_id + } + + /// Returns the exact private-evaluation basis identity. + #[must_use] + pub const fn evaluation_basis_id(&self) -> EchoOperationEvaluationBasisIdV1 { + self.evaluation_basis_id + } + + /// Returns the bounded private-evaluation identity. + #[must_use] + pub const fn private_evaluation_id(&self) -> EchoOperationPrivateEvaluationIdV1 { + self.private_evaluation_id + } + + /// Returns the rejected preparation's patch digest. + #[must_use] + pub const fn prepared_patch_digest(&self) -> Hash { + self.prepared_patch_digest + } + + /// Returns the rejected preparation's typed result identity. + #[must_use] + pub const fn prepared_result_id(&self) -> EchoOperationResultIdV1 { + self.prepared_result_id + } + + /// Returns the evaluator-recorded actual-footprint identity. + #[must_use] + pub const fn actual_footprint_digest(&self) -> Hash { + self.actual_footprint_digest + } + + /// Returns the complete rejected preparation's identity. + #[must_use] + pub const fn preparation_id(&self) -> PreparedEchoOperationIdV1 { + self.preparation_id + } + + /// Returns canonical indices of earlier applied Tick members which blocked + /// this preparation. + #[must_use] + pub fn blocked_by(&self) -> &[u32] { + &self.blocked_by + } + + /// Returns the deciding Tick coordinate once this conflict becomes a + /// terminal Action outcome. + #[must_use] + pub const fn decision_coordinate(&self) -> Option { + match &self.decision_coordinate { + Some(coordinate) => Some(**coordinate), + None => None, + } + } +} + +/// Typed terminal outcome for one executable-operation Action selected by a +/// scheduler-owned Tick. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EchoOperationActionOutcomeV1 { + /// The Action's prepared member consequence entered the composite Tick. + Committed(Box), + /// Private bounded evaluation refused the Action and emitted no mutation. + Obstructed(EchoOperationObstructionV1), + /// An earlier applied Action in the same Tick reserved an overlapping + /// footprint. + RejectedFootprintConflict(Box), +} + +impl EchoOperationActionOutcomeV1 { + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_obstruction_kind_for_test( + &mut self, + kind: EchoOperationObstructionKindV1, + ) { + if let Self::Obstructed(obstruction) = self { + obstruction.kind = kind; + } + } + + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_conflict_preparation_id_for_test(&mut self, digest: Hash) { + if let Self::RejectedFootprintConflict(conflict) = self { + conflict.preparation_id = PreparedEchoOperationIdV1(digest); + } + } + + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_conflict_blockers_for_test(&mut self, replacement: Vec) { + if let Self::RejectedFootprintConflict(conflict) = self { + conflict.blocked_by = replacement; + } + } + + #[cfg(any(test, feature = "host_test"))] + pub(crate) fn replace_decision_member_index_for_test(&mut self, replacement: u32) { + let coordinate = match self { + Self::Committed(_) => None, + Self::Obstructed(obstruction) => obstruction.decision_coordinate.as_deref_mut(), + Self::RejectedFootprintConflict(conflict) => { + conflict.decision_coordinate.as_deref_mut() + } + }; + if let Some(coordinate) = coordinate { + coordinate.member_index = replacement; + } + } +} + +pub(crate) fn retain_action_outcome_v1( + submission_id: Hash, + ingress_id: Hash, + outcome: &EchoOperationActionOutcomeV1, +) -> Result, EchoOperationArtifactErrorV1> { + let mut out = Vec::new(); + out.extend_from_slice(ACTION_OUTCOME_RECORD_MAGIC); + out.extend_from_slice(&submission_id); + out.extend_from_slice(&ingress_id); + match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + out.push(1); + let receipt_bytes = receipt.to_canonical_bytes()?; + let receipt_len = u64::try_from(receipt_bytes.len()) + .map_err(|_| invalid_structure("Action receipt length is not representable"))?; + out.extend_from_slice(&receipt_len.to_le_bytes()); + out.extend_from_slice(&receipt_bytes); + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + out.push(2); + out.push(obstruction_kind_code(obstruction.kind)); + out.extend_from_slice(&obstruction.package_id.as_hash()); + out.extend_from_slice(&obstruction.installed_operation_id.as_hash()); + out.extend_from_slice(&obstruction.invocation_admission.policy_id); + out.extend_from_slice( + &obstruction + .invocation_admission + .maximum_budget + .steps + .to_le_bytes(), + ); + out.extend_from_slice( + &obstruction + .invocation_admission + .maximum_budget + .read_bytes + .to_le_bytes(), + ); + out.extend_from_slice( + &obstruction + .invocation_admission + .maximum_budget + .write_bytes + .to_le_bytes(), + ); + out.extend_from_slice(&obstruction.invocation_admission.admission_id.as_hash()); + out.extend_from_slice(&obstruction.invocation_id.as_hash()); + out.extend_from_slice(&obstruction.evaluation_basis_id.as_hash()); + retain_action_decision_coordinate_v1( + &mut out, + obstruction.decision_coordinate().ok_or_else(|| { + invalid_structure("terminal Action obstruction has no deciding Tick") + })?, + ); + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + out.push(3); + out.extend_from_slice(&conflict.package_id.as_hash()); + out.extend_from_slice(&conflict.installed_operation_id.as_hash()); + out.extend_from_slice(&conflict.invocation_admission_policy_id); + out.extend_from_slice( + &conflict + .invocation_admission_maximum_budget + .steps + .to_le_bytes(), + ); + out.extend_from_slice( + &conflict + .invocation_admission_maximum_budget + .read_bytes + .to_le_bytes(), + ); + out.extend_from_slice( + &conflict + .invocation_admission_maximum_budget + .write_bytes + .to_le_bytes(), + ); + out.extend_from_slice(&conflict.invocation_admission_id.as_hash()); + out.extend_from_slice(&conflict.invocation_id.as_hash()); + out.extend_from_slice(&conflict.evaluation_basis_id.as_hash()); + out.extend_from_slice(&conflict.private_evaluation_id.as_hash()); + out.extend_from_slice(&conflict.prepared_patch_digest); + out.extend_from_slice(&conflict.prepared_result_id.as_hash()); + out.extend_from_slice(&conflict.actual_footprint_digest); + out.extend_from_slice(&conflict.preparation_id.as_hash()); + retain_action_decision_coordinate_v1( + &mut out, + conflict.decision_coordinate().ok_or_else(|| { + invalid_structure("terminal Action conflict has no deciding Tick") + })?, + ); + let blocker_count = u64::try_from(conflict.blocked_by.len()) + .map_err(|_| invalid_structure("Action blocker count is not representable"))?; + out.extend_from_slice(&blocker_count.to_le_bytes()); + for blocker in &conflict.blocked_by { + out.extend_from_slice(&blocker.to_le_bytes()); + } + } + } + Ok(out) +} + +fn retain_action_decision_coordinate_v1( + out: &mut Vec, + coordinate: EchoOperationActionDecisionCoordinateV1, +) { + out.extend_from_slice(coordinate.writer_head.worldline_id.as_bytes()); + out.extend_from_slice(coordinate.writer_head.head_id.as_bytes()); + out.extend_from_slice(&coordinate.worldline_tick_after.as_u64().to_le_bytes()); + out.extend_from_slice(&coordinate.commit_global_tick.as_u64().to_le_bytes()); + out.extend_from_slice(&coordinate.commit_id); + out.extend_from_slice(&coordinate.tick_receipt_digest); + out.extend_from_slice(&u64::from(coordinate.member_index).to_le_bytes()); } -impl EchoOperationObstructionV1 { - /// Returns the stable obstruction category. - #[must_use] - pub const fn kind(&self) -> EchoOperationObstructionKindV1 { - self.kind - } +fn recover_action_decision_coordinate_v1( + bytes: &[u8], + offset: &mut usize, +) -> Result { + let worldline_id = crate::WorldlineId::from_bytes(read_action_outcome_hash(bytes, offset)?); + let head_id = HeadId::from_bytes(read_action_outcome_hash(bytes, offset)?); + let worldline_tick_after = WorldlineTick::from_raw(read_action_outcome_u64(bytes, offset)?); + let commit_global_tick = GlobalTick::from_raw(read_action_outcome_u64(bytes, offset)?); + let commit_id = read_action_outcome_hash(bytes, offset)?; + let tick_receipt_digest = read_action_outcome_hash(bytes, offset)?; + let member_index = u32::try_from(read_action_outcome_u64(bytes, offset)?) + .map_err(|_| invalid_structure("Action Tick member index is not representable"))?; + Ok(EchoOperationActionDecisionCoordinateV1 { + writer_head: WriterHeadKey { + worldline_id, + head_id, + }, + worldline_tick_after, + commit_global_tick, + commit_id, + tick_receipt_digest, + member_index, + }) +} - /// Returns the exact installed operation against which evaluation was attempted. - #[must_use] - pub const fn installed_operation_id(&self) -> InstalledEchoOperationIdV1 { - self.installed_operation_id +pub(crate) fn recover_action_outcome_v1( + bytes: &[u8], +) -> Result<(Hash, Hash, EchoOperationActionOutcomeV1), EchoOperationArtifactErrorV1> { + let minimum = ACTION_OUTCOME_RECORD_MAGIC.len() + 32 + 32 + 1; + if bytes.len() < minimum + || &bytes[..ACTION_OUTCOME_RECORD_MAGIC.len()] != ACTION_OUTCOME_RECORD_MAGIC + { + return Err(invalid_structure( + "executable-operation Action outcome record has invalid framing", + )); } - - /// Returns the exact runtime admission that authorized this evaluation attempt. - #[must_use] - pub const fn invocation_admission_id(&self) -> EchoOperationInvocationAdmissionIdV1 { - self.invocation_admission_id + let mut offset = ACTION_OUTCOME_RECORD_MAGIC.len(); + let submission_id = read_action_outcome_hash(bytes, &mut offset)?; + let ingress_id = read_action_outcome_hash(bytes, &mut offset)?; + let tag = *bytes + .get(offset) + .ok_or_else(|| invalid_structure("Action outcome tag is missing"))?; + offset += 1; + let outcome = match tag { + 1 => { + let len = read_action_outcome_u64(bytes, &mut offset)?; + let len = usize::try_from(len) + .map_err(|_| invalid_structure("Action receipt length is not representable"))?; + let end = offset + .checked_add(len) + .ok_or_else(|| invalid_structure("Action receipt length overflows"))?; + let receipt_bytes = bytes + .get(offset..end) + .ok_or_else(|| invalid_structure("Action receipt bytes are truncated"))?; + offset = end; + EchoOperationActionOutcomeV1::Committed(Box::new( + EchoOperationReceiptV1::from_action_batch_canonical_bytes(receipt_bytes)?, + )) + } + 2 => { + let kind_code = *bytes + .get(offset) + .ok_or_else(|| invalid_structure("Action obstruction kind is missing"))?; + offset += 1; + EchoOperationActionOutcomeV1::Obstructed(EchoOperationObstructionV1 { + kind: obstruction_kind_from_code(kind_code)?, + package_id: EchoOperationPackageIdV1(read_action_outcome_hash(bytes, &mut offset)?), + installed_operation_id: InstalledEchoOperationIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: read_action_outcome_hash(bytes, &mut offset)?, + maximum_budget: EchoOperationBudgetV1 { + steps: read_action_outcome_u64(bytes, &mut offset)?, + read_bytes: read_action_outcome_u64(bytes, &mut offset)?, + write_bytes: read_action_outcome_u64(bytes, &mut offset)?, + }, + admission_id: EchoOperationInvocationAdmissionIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + }), + invocation_id: EchoOperationInvocationIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + evaluation_basis_id: EchoOperationEvaluationBasisIdV1(read_action_outcome_hash( + bytes, + &mut offset, + )?), + decision_coordinate: Some(Box::new(recover_action_decision_coordinate_v1( + bytes, + &mut offset, + )?)), + }) + } + 3 => { + let package_id = + EchoOperationPackageIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let installed_operation_id = + InstalledEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let invocation_admission_policy_id = read_action_outcome_hash(bytes, &mut offset)?; + let invocation_admission_maximum_budget = EchoOperationBudgetV1 { + steps: read_action_outcome_u64(bytes, &mut offset)?, + read_bytes: read_action_outcome_u64(bytes, &mut offset)?, + write_bytes: read_action_outcome_u64(bytes, &mut offset)?, + }; + let invocation_admission_id = + EchoOperationInvocationAdmissionIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let invocation_id = + EchoOperationInvocationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let evaluation_basis_id = + EchoOperationEvaluationBasisIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let private_evaluation_id = + EchoOperationPrivateEvaluationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let prepared_patch_digest = read_action_outcome_hash(bytes, &mut offset)?; + let prepared_result_id = + EchoOperationResultIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let actual_footprint_digest = read_action_outcome_hash(bytes, &mut offset)?; + let preparation_id = + PreparedEchoOperationIdV1(read_action_outcome_hash(bytes, &mut offset)?); + let decision_coordinate = recover_action_decision_coordinate_v1(bytes, &mut offset)?; + let blocker_count = read_action_outcome_u64(bytes, &mut offset)?; + let maximum_encoded_blockers = + u64::try_from(bytes.len().saturating_sub(offset) / core::mem::size_of::()) + .unwrap_or(u64::MAX); + if blocker_count > maximum_encoded_blockers { + return Err(invalid_structure("Action blocker bytes are truncated")); + } + let blocker_count = usize::try_from(blocker_count) + .map_err(|_| invalid_structure("Action blocker count is not representable"))?; + let mut blocked_by = Vec::with_capacity(blocker_count); + for _ in 0..blocker_count { + let end = offset + .checked_add(4) + .ok_or_else(|| invalid_structure("Action blocker offset overflows"))?; + let raw: [u8; 4] = bytes + .get(offset..end) + .ok_or_else(|| invalid_structure("Action blocker bytes are truncated"))? + .try_into() + .map_err(|_| invalid_structure("Action blocker is not four bytes"))?; + offset = end; + blocked_by.push(u32::from_le_bytes(raw)); + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(Box::new( + EchoOperationFootprintConflictV1 { + package_id, + installed_operation_id, + invocation_admission_policy_id, + invocation_admission_maximum_budget, + invocation_admission_id, + invocation_id, + evaluation_basis_id, + private_evaluation_id, + prepared_patch_digest, + prepared_result_id, + actual_footprint_digest, + preparation_id, + blocked_by, + decision_coordinate: Some(Box::new(decision_coordinate)), + }, + )) + } + _ => { + return Err(invalid_structure( + "unknown executable-operation Action outcome tag", + )) + } + }; + if offset != bytes.len() { + return Err(invalid_structure( + "executable-operation Action outcome record has trailing bytes", + )); } + Ok((submission_id, ingress_id, outcome)) +} - /// Returns the identity of this typed no-parent-patch obstruction. - #[must_use] - pub fn identity(&self) -> EchoOperationObstructionIdV1 { - let mut hasher = Hasher::new(); - hasher.update(OBSTRUCTION_ID_DOMAIN); - hasher.update(&self.package_id.as_hash()); - hasher.update(&self.installed_operation_id.as_hash()); - hasher.update(&self.invocation_admission_id.as_hash()); - hasher.update(&self.invocation_id.as_hash()); - hasher.update(&self.evaluation_basis_id.as_hash()); - hasher.update(&[obstruction_kind_code(self.kind)]); - EchoOperationObstructionIdV1(hasher.finalize().into()) +fn read_action_outcome_hash( + bytes: &[u8], + offset: &mut usize, +) -> Result { + let end = offset + .checked_add(32) + .ok_or_else(|| invalid_structure("Action outcome hash offset overflows"))?; + let value = bytes + .get(*offset..end) + .ok_or_else(|| invalid_structure("Action outcome hash is truncated"))? + .try_into() + .map_err(|_| invalid_structure("Action outcome hash is not 32 bytes"))?; + *offset = end; + Ok(value) +} + +fn read_action_outcome_u64( + bytes: &[u8], + offset: &mut usize, +) -> Result { + let end = offset + .checked_add(8) + .ok_or_else(|| invalid_structure("Action outcome integer offset overflows"))?; + let raw: [u8; 8] = bytes + .get(*offset..end) + .ok_or_else(|| invalid_structure("Action outcome integer is truncated"))? + .try_into() + .map_err(|_| invalid_structure("Action outcome integer is not eight bytes"))?; + *offset = end; + Ok(u64::from_le_bytes(raw)) +} + +fn obstruction_kind_from_code( + code: u8, +) -> Result { + match code { + 1 => Ok(EchoOperationObstructionKindV1::OperationUnavailable), + 2 => Ok(EchoOperationObstructionKindV1::BasisChanged), + 3 => Ok(EchoOperationObstructionKindV1::BudgetExceeded), + 4 => Ok(EchoOperationObstructionKindV1::NodeMissing), + 5 => Ok(EchoOperationObstructionKindV1::NodeTypeMismatch), + 6 => Ok(EchoOperationObstructionKindV1::AttachmentMissing), + 7 => Ok(EchoOperationObstructionKindV1::AttachmentNotAtom), + 8 => Ok(EchoOperationObstructionKindV1::AttachmentTypeMismatch), + 9 => Ok(EchoOperationObstructionKindV1::PreconditionMismatch), + 10 => Ok(EchoOperationObstructionKindV1::FootprintViolation), + 11 => Ok(EchoOperationObstructionKindV1::ReplacementTooLarge), + 12 => Ok(EchoOperationObstructionKindV1::EvaluationAuthorityMismatch), + _ => Err(invalid_structure( + "unknown executable-operation obstruction kind", + )), } } @@ -2590,6 +3520,14 @@ impl PreparedEchoOperationV1 { self.invocation_admission_id } + pub(crate) const fn invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission_policy_id + } + + pub(crate) const fn invocation_admission_maximum_budget(&self) -> EchoOperationBudgetV1 { + self.invocation_admission_maximum_budget + } + /// Returns the declared-footprint identity bound by evaluation. #[must_use] pub const fn declared_footprint_digest(&self) -> Hash { @@ -2632,6 +3570,10 @@ impl PreparedEchoOperationV1 { self.result_id } + pub(crate) fn prepared_patch_digest(&self) -> Hash { + self.patch.digest() + } + pub(crate) const fn package_id(&self) -> EchoOperationPackageIdV1 { self.installed.package_id } @@ -2640,6 +3582,10 @@ impl PreparedEchoOperationV1 { self.installed.installed_operation_id } + pub(crate) const fn invocation_id(&self) -> EchoOperationInvocationIdV1 { + self.invocation_id + } + pub(crate) fn is_owned_by(&self, authority: &EchoOperationEvaluationAuthorityV1) -> bool { self.evaluation_authority.same_owner(authority) } @@ -2660,9 +3606,14 @@ pub(crate) fn prepare_operation_v1( kind, package_id, installed_operation_id: admitted.installed_operation_id, - invocation_admission_id: admitted.admission_id, + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: admitted.admission_policy_id, + maximum_budget: admitted.admission_policy.maximum_delegated_budget, + admission_id: admitted.admission_id, + }), invocation_id, evaluation_basis_id: admitted.invocation.evaluation_basis.identity(), + decision_coordinate: None, }) }; let Some(installed) = installed else { @@ -3056,6 +4007,16 @@ impl EchoOperationReceiptV1 { self.invocation_admission_id } + pub(crate) const fn retained_invocation_admission_policy_id(&self) -> Hash { + self.invocation_admission_policy_id + } + + pub(crate) const fn retained_invocation_admission_maximum_budget( + &self, + ) -> EchoOperationBudgetV1 { + self.invocation_admission_maximum_budget + } + /// Returns the bounded private-evaluation identity. #[must_use] pub const fn private_evaluation_id(&self) -> EchoOperationPrivateEvaluationIdV1 { @@ -3092,12 +4053,21 @@ impl EchoOperationReceiptV1 { self.prepared_patch_digest } - /// Returns singleton composition evidence only for a committed consequence. + /// Returns composition evidence only for a committed consequence. + /// + /// Direct compatibility commits bind singleton composition. Scheduler-owned + /// Action receipts bind the exact composite Tick consequence. #[must_use] pub const fn composition_digest(&self) -> Option { self.composition_digest } + /// Replaces composition evidence for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_composition_digest_for_test(&mut self, composition_digest: Hash) { + self.composition_digest = Some(composition_digest); + } + /// Returns the closed terminal-outcome identity. #[must_use] pub const fn terminal_outcome_digest(&self) -> Hash { @@ -3289,6 +4259,19 @@ impl EchoOperationReceiptV1 { } pub(crate) fn from_canonical_bytes(bytes: &[u8]) -> Result { + Self::decode_canonical_bytes(bytes, false) + } + + fn from_action_batch_canonical_bytes( + bytes: &[u8], + ) -> Result { + Self::decode_canonical_bytes(bytes, true) + } + + fn decode_canonical_bytes( + bytes: &[u8], + allow_composite_context: bool, + ) -> Result { let value = decode_canonical_cbor_v1(bytes).map_err(canonical_error)?; let mut fields = exact_text_map( value, @@ -3552,9 +4535,12 @@ impl EchoOperationReceiptV1 { || receipt.commit_id == [0; 32] || receipt.tick_receipt_digest == [0; 32] || receipt.committed_patch_digest.is_none() - || receipt.committed_patch_digest != Some(receipt.prepared_patch_digest) || receipt.committed_result_id != Some(receipt.prepared_result_id) - || receipt.composition_digest != Some(expected_composition_digest) + || receipt.composition_digest.is_none() + || (receipt.committed_patch_digest != Some(receipt.prepared_patch_digest) + && !allow_composite_context) + || (receipt.committed_patch_digest == Some(receipt.prepared_patch_digest) + && receipt.composition_digest != Some(expected_composition_digest)) || receipt.state_root_before != receipt.evaluation_basis.state_root || committed_worldline_tick_after != Some(receipt.worldline_tick_after) => { @@ -3701,6 +4687,20 @@ pub enum EchoOperationCommitErrorV1 { /// The worldline-local transaction coordinate cannot advance. #[error("executable-operation transaction coordinate overflow")] TransactionCoordinateOverflow, + /// The candidate count cannot be represented by Tick blocker indices. + #[error("executable-operation Action batch has too many candidates")] + TooManyCandidates, + /// More than one candidate claimed the same canonical ingress identity. + #[error("executable-operation Action batch contains duplicate ingress identity")] + DuplicateCandidateIngress, + /// Test-only scheduler fault injected before Action Tick construction. + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + #[error("injected executable-operation Action Tick construction failure")] + InjectedTickConstructionFailure, } pub(crate) struct EchoOperationCommitMaterialV1 { @@ -3710,6 +4710,45 @@ pub(crate) struct EchoOperationCommitMaterialV1 { pub patch: WarpTickPatchV1, } +pub(crate) struct SchedulerEchoOperationCandidateV1 { + pub submission_id: Hash, + pub ingress_id: Hash, + pub scope: NodeKey, + pub rule_id: Hash, + pub preparation: EchoOperationPreparationV1, +} + +pub(crate) struct EchoOperationActionBatchCommitMaterialV1 { + pub snapshot: Snapshot, + pub tick_receipt: TickReceipt, + pub patch: WarpTickPatchV1, + pub outcomes: Vec<(Hash, EchoOperationActionOutcomeV1)>, +} + +enum SchedulerEchoOperationDecisionV1 { + Applied(Box), + Obstructed(EchoOperationObstructionV1), + RejectedFootprintConflict(Box), +} + +fn scheduler_composition_budget_obstruction_v1( + prepared: &PreparedEchoOperationV1, +) -> EchoOperationObstructionV1 { + EchoOperationObstructionV1 { + kind: EchoOperationObstructionKindV1::BudgetExceeded, + package_id: prepared.package_id(), + installed_operation_id: prepared.installed_operation_id(), + invocation_admission: Box::new(EchoOperationObstructionAdmissionEvidenceV1 { + policy_id: prepared.invocation_admission_policy_id(), + maximum_budget: prepared.invocation_admission_maximum_budget(), + admission_id: prepared.invocation_admission_id(), + }), + invocation_id: prepared.invocation_id(), + evaluation_basis_id: prepared.evaluation_basis().identity(), + decision_coordinate: None, + } +} + #[derive(Clone, Copy, Debug)] struct EchoOperationTerminalMaterialV1 { posture: EchoOperationTerminalPostureV1, @@ -3721,6 +4760,406 @@ struct EchoOperationTerminalMaterialV1 { worldline_tick_after: WorldlineTick, } +pub(crate) fn commit_scheduler_action_batch_to_state_v1( + mut candidates: Vec, + writer_head: WriterHeadKey, + state: &mut WorldlineState, + commit_global_tick: GlobalTick, + policy_id: u32, +) -> Result { + if candidates.len() > ACTION_BATCH_CANDIDATE_LIMIT_V1 { + return Err(EchoOperationCommitErrorV1::TooManyCandidates); + } + // This order is part of the receipt-correlation contract: coordinator + // outcome lookup maps the same Tick's ingress-sorted correlations onto + // these receipt entries by position. + candidates.sort_by_key(|candidate| candidate.ingress_id); + if candidates + .windows(2) + .any(|pair| pair[0].ingress_id == pair[1].ingress_id) + { + return Err(EchoOperationCommitErrorV1::DuplicateCandidateIngress); + } + let state_root_before = state.state_root(); + let tx_raw = state + .current_tick() + .as_u64() + .checked_add(1) + .ok_or(EchoOperationCommitErrorV1::TransactionCoordinateOverflow)?; + let tx = TxId::from_raw(tx_raw); + + let mut entries = Vec::with_capacity(candidates.len()); + let mut blocked_by = Vec::with_capacity(candidates.len()); + let mut decisions = Vec::with_capacity(candidates.len()); + let mut accepted_footprints: Vec<(u32, Footprint)> = Vec::new(); + let mut accepted_operation_count = 0_usize; + let mut ordered_rule_ids = Vec::with_capacity(candidates.len()); + let mut footprint_comparisons = 0_usize; + let mut blocker_evidence_count = 0_usize; + + for (entry_index, candidate) in candidates.into_iter().enumerate() { + let entry_index_u32 = u32::try_from(entry_index) + .map_err(|_| EchoOperationCommitErrorV1::TooManyCandidates)?; + ordered_rule_ids.push(candidate.rule_id); + let scope_hash = crate::scope_hash(&candidate.rule_id, &candidate.scope); + match candidate.preparation { + EchoOperationPreparationV1::Obstructed(obstruction) => { + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ), + }); + blocked_by.push(Vec::new()); + decisions.push(( + candidate.submission_id, + SchedulerEchoOperationDecisionV1::Obstructed(obstruction), + )); + } + EchoOperationPreparationV1::Prepared(prepared) => { + let comparison_cost = accepted_footprints.len(); + let comparison_budget_exceeded = footprint_comparisons + .checked_add(comparison_cost) + .is_none_or(|total| total > ACTION_BATCH_FOOTPRINT_COMPARISON_LIMIT_V1); + let blockers = if comparison_budget_exceeded { + Vec::new() + } else { + footprint_comparisons += comparison_cost; + accepted_footprints + .iter() + .filter_map(|(accepted_index, footprint)| { + crate::engine_impl::footprints_conflict( + prepared.actual_footprint(), + footprint, + ) + .then_some(*accepted_index) + }) + .collect::>() + }; + let blocker_budget_exceeded = blocker_evidence_count + .checked_add(blockers.len()) + .is_none_or(|total| total > ACTION_BATCH_BLOCKER_EVIDENCE_LIMIT_V1); + let operation_budget_exceeded = blockers.is_empty() + && accepted_operation_count + .checked_add(prepared.patch().ops().len()) + .is_none_or(|total| total > ACTION_BATCH_OPERATION_LIMIT_V1); + if comparison_budget_exceeded + || blocker_budget_exceeded + || operation_budget_exceeded + { + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ), + }); + blocked_by.push(Vec::new()); + decisions.push(( + candidate.submission_id, + SchedulerEchoOperationDecisionV1::Obstructed( + scheduler_composition_budget_obstruction_v1(&prepared), + ), + )); + continue; + } + if blockers.is_empty() { + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Applied, + }); + blocked_by.push(Vec::new()); + accepted_footprints + .push((entry_index_u32, prepared.actual_footprint().clone())); + accepted_operation_count += prepared.patch().ops().len(); + decisions.push(( + candidate.submission_id, + SchedulerEchoOperationDecisionV1::Applied(prepared), + )); + } else { + blocker_evidence_count += blockers.len(); + entries.push(TickReceiptEntry { + rule_id: candidate.rule_id, + scope_hash, + scope: candidate.scope, + disposition: TickReceiptDisposition::Rejected( + TickReceiptRejection::FootprintConflict, + ), + }); + blocked_by.push(blockers.clone()); + decisions.push(( + candidate.submission_id, + SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(Box::new( + EchoOperationFootprintConflictV1 { + package_id: prepared.package_id(), + installed_operation_id: prepared.installed_operation_id(), + invocation_admission_policy_id: prepared + .invocation_admission_policy_id(), + invocation_admission_maximum_budget: prepared + .invocation_admission_maximum_budget(), + invocation_admission_id: prepared.invocation_admission_id(), + invocation_id: prepared.invocation_id(), + evaluation_basis_id: prepared.evaluation_basis().identity(), + private_evaluation_id: prepared.private_evaluation_id(), + prepared_patch_digest: prepared.prepared_patch_digest(), + prepared_result_id: prepared.result_id(), + actual_footprint_digest: prepared.actual_footprint_digest(), + preparation_id: prepared.preparation_id(), + blocked_by: blockers, + decision_coordinate: None, + }, + )), + )); + } + } + } + } + + let tick_receipt = TickReceipt::new(tx, entries, blocked_by); + let plan_digest = action_batch_plan_digest(tick_receipt.entries()); + let applied = decisions + .iter() + .filter_map(|(_, decision)| match decision { + SchedulerEchoOperationDecisionV1::Applied(prepared) => Some(prepared.as_ref()), + SchedulerEchoOperationDecisionV1::Obstructed(_) + | SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(_) => None, + }) + .collect::>(); + let patch = action_batch_patch_from_preparations_v1(policy_id, &ordered_rule_ids, &applied); + let rewrites_digest = action_batch_rewrites_digest(&applied); + let composition_digest = action_batch_composition_digest(&applied, patch.digest()); + + let mut next_state = state.warp_state.clone(); + patch.apply_to_state(&mut next_state)?; + let state_root_after = + crate::snapshot::compute_state_root_for_warp_state(&next_state, state.root()); + let parents = state + .last_snapshot + .as_ref() + .map(|snapshot| vec![snapshot.hash]) + .unwrap_or_default(); + let commit_id = compute_commit_hash_v2( + &state_root_after, + &parents, + &patch.digest(), + patch.policy_id(), + ); + let snapshot = Snapshot { + root: *state.root(), + hash: commit_id, + state_root: state_root_after, + parents, + plan_digest, + decision_digest: tick_receipt.digest(), + rewrites_digest, + patch_digest: patch.digest(), + policy_id: patch.policy_id(), + tx, + }; + + debug_assert_action_member_alignment_v1(decisions.len(), tick_receipt.entries().len()); + let outcomes = decisions + .into_iter() + .enumerate() + .map( + |(member_index, (submission_id, decision))| -> Result<_, EchoOperationCommitErrorV1> { + let member_index = u32::try_from(member_index) + .map_err(|_| EchoOperationCommitErrorV1::TooManyCandidates)?; + let decision_coordinate = EchoOperationActionDecisionCoordinateV1 { + writer_head, + worldline_tick_after: WorldlineTick::from_raw(tx_raw), + commit_global_tick, + commit_id, + tick_receipt_digest: tick_receipt.digest(), + member_index, + }; + let outcome = match decision { + SchedulerEchoOperationDecisionV1::Applied(prepared) => { + let mut receipt = build_receipt( + &prepared, + EchoOperationTerminalMaterialV1 { + posture: EchoOperationTerminalPostureV1::Committed, + state_root_before, + state_root_after, + commit_id, + tick_receipt_digest: tick_receipt.digest(), + commit_global_tick: Some(commit_global_tick), + worldline_tick_after: WorldlineTick::from_raw(tx_raw), + }, + ); + receipt.committed_patch_digest = Some(patch.digest()); + receipt.composition_digest = Some(composition_digest); + receipt.terminal_outcome_digest = terminal_outcome_digest(&receipt); + receipt.receipt_digest = receipt_digest(&receipt); + EchoOperationActionOutcomeV1::Committed(Box::new(receipt)) + } + SchedulerEchoOperationDecisionV1::Obstructed(mut obstruction) => { + obstruction.decision_coordinate = Some(Box::new(decision_coordinate)); + EchoOperationActionOutcomeV1::Obstructed(obstruction) + } + SchedulerEchoOperationDecisionV1::RejectedFootprintConflict(mut conflict) => { + conflict.decision_coordinate = Some(Box::new(decision_coordinate)); + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) + } + }; + Ok((submission_id, outcome)) + }, + ) + .collect::, _>>()?; + + state.warp_state = next_state; + state.last_snapshot = Some(snapshot.clone()); + state + .tick_history + .push((snapshot.clone(), tick_receipt.clone(), patch.clone())); + state.tx_counter = tx_raw; + + Ok(EchoOperationActionBatchCommitMaterialV1 { + snapshot, + tick_receipt, + patch, + outcomes, + }) +} + +fn debug_assert_action_member_alignment_v1(decision_count: usize, receipt_entry_count: usize) { + debug_assert_eq!( + decision_count, receipt_entry_count, + "Action member index is the Tick receipt entry index" + ); +} + +pub(crate) fn action_batch_patch_from_preparations_v1( + policy_id: u32, + ordered_rule_ids: &[Hash], + prepared: &[&PreparedEchoOperationV1], +) -> WarpTickPatchV1 { + let mut in_slots = Vec::new(); + let mut out_slots = Vec::new(); + let mut ops = Vec::new(); + for member in prepared { + in_slots.extend_from_slice(member.patch().in_slots()); + out_slots.extend_from_slice(member.patch().out_slots()); + ops.extend_from_slice(member.patch().ops()); + } + WarpTickPatchV1::new( + policy_id, + action_batch_rule_pack_id(ordered_rule_ids), + TickCommitStatus::Committed, + in_slots, + out_slots, + ops, + ) +} + +fn action_batch_rule_pack_id(rule_ids: &[Hash]) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_RULE_PACK_DOMAIN); + hasher.update(&(rule_ids.len() as u64).to_le_bytes()); + for rule_id in rule_ids { + hasher.update(rule_id); + } + hasher.finalize().into() +} + +fn action_batch_plan_digest(entries: &[TickReceiptEntry]) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_PLAN_DOMAIN); + hasher.update(&(entries.len() as u64).to_le_bytes()); + for entry in entries { + hasher.update(&entry.rule_id); + hasher.update(&entry.scope_hash); + } + hasher.finalize().into() +} + +fn action_batch_rewrites_digest(prepared: &[&PreparedEchoOperationV1]) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_REWRITES_DOMAIN); + hasher.update(&(prepared.len() as u64).to_le_bytes()); + for member in prepared { + hasher.update(&member.preparation_id().as_hash()); + } + hasher.finalize().into() +} + +fn action_batch_composition_digest( + prepared: &[&PreparedEchoOperationV1], + committed_patch_digest: Hash, +) -> Hash { + action_batch_composition_digest_from_parts( + committed_patch_digest, + prepared.len(), + prepared.iter().map(|member| { + ( + member.preparation_id(), + member.patch().digest(), + member.result_id(), + member.evaluation_basis().identity(), + member.actual_footprint_digest(), + ) + }), + ) +} + +pub(crate) fn action_batch_composition_digest_from_receipts_v1( + receipts: &[&EchoOperationReceiptV1], + committed_patch_digest: Hash, +) -> Hash { + action_batch_composition_digest_from_parts( + committed_patch_digest, + receipts.len(), + receipts.iter().map(|receipt| { + ( + receipt.preparation_id, + receipt.prepared_patch_digest, + receipt.prepared_result_id, + receipt.evaluation_basis_id, + receipt.actual_footprint_digest, + ) + }), + ) +} + +fn action_batch_composition_digest_from_parts( + committed_patch_digest: Hash, + member_count: usize, + members: impl IntoIterator< + Item = ( + PreparedEchoOperationIdV1, + Hash, + EchoOperationResultIdV1, + EchoOperationEvaluationBasisIdV1, + Hash, + ), + >, +) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ACTION_BATCH_COMPOSITION_DOMAIN); + hasher.update(&committed_patch_digest); + hasher.update(&(member_count as u64).to_le_bytes()); + for ( + preparation_id, + prepared_patch_digest, + prepared_result_id, + evaluation_basis_id, + actual_footprint_digest, + ) in members + { + hasher.update(&preparation_id.as_hash()); + hasher.update(&prepared_patch_digest); + hasher.update(&prepared_result_id.as_hash()); + hasher.update(&evaluation_basis_id.as_hash()); + hasher.update(&actual_footprint_digest); + } + hasher.finalize().into() +} + pub(crate) fn not_committed_basis_changed( prepared: &PreparedEchoOperationV1, current_state_root: Hash, @@ -4682,6 +6121,13 @@ mod tests { } } + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "Action member index is the Tick receipt entry index")] + fn action_member_alignment_rejects_divergent_decision_and_receipt_counts() { + debug_assert_action_member_alignment_v1(2, 1); + } + #[test] fn footprint_digest_is_prefix_free_across_adjacent_sets() { let mut read_node_bytes = [0; 64]; @@ -4742,9 +6188,10 @@ mod tests { let mut footprint = Footprint::default(); footprint.b_out.insert(node.warp_id, 9); footprints.push(footprint); - let mut footprint = Footprint::default(); - footprint.factor_mask = 1; - footprints.push(footprint); + footprints.push(Footprint { + factor_mask: 1, + ..Footprint::default() + }); let digests = footprints.iter().map(footprint_digest).collect::>(); for (index, digest) in digests.iter().enumerate() { @@ -4856,7 +6303,8 @@ mod tests { }, child_store, ); - let state = WorldlineState::new(warp_state, root).expect("the descended fixture is lawful"); + let mut state = + WorldlineState::new(warp_state, root).expect("the descended fixture is lawful"); let operation_coordinate = "echo.fixture.DescendedCreateIfAbsent.v1"; let authority_profile = digest(40); @@ -4969,8 +6417,7 @@ mod tests { "each portal pointer read must be charged as a bounded evaluator step" ); - let mut budget_limited_state = state.clone(); - budget_limited_state + state .warp_state .store_mut(&root_warp) .expect("the fixture retains its root store") @@ -4985,7 +6432,7 @@ mod tests { writer_head, WorldlineTick::ZERO, None, - budget_limited_state.state_root(), + state.state_root(), digest(51), echo_operation_anchored_node_absent_application_basis_v1(target), ); @@ -5010,7 +6457,7 @@ mod tests { .to_canonical_bytes() .expect("budget-limited invocation encodes"), budget_limited_basis, - &budget_limited_state, + &state, evaluation_authority.clone(), ) .expect("the budget-limited invocation admits"); @@ -5018,7 +6465,7 @@ mod tests { Some(&installed), admitted, budget_limited_basis, - &budget_limited_state, + &state, crate::POLICY_ID_NO_POLICY_V0, &evaluation_authority, ) else { @@ -5132,6 +6579,47 @@ mod tests { encode_canonical_cbor_v1(&CanonicalValueV1::Map(fields)).expect("fixture map encodes") } + #[test] + fn retained_action_outcome_rejects_impossible_blocker_count_before_allocation() { + let mut bytes = Vec::new(); + bytes.extend_from_slice(ACTION_OUTCOME_RECORD_MAGIC); + bytes.extend_from_slice(&digest(1)); + bytes.extend_from_slice(&digest(2)); + bytes.push(3); + bytes.extend_from_slice(&digest(3)); + bytes.extend_from_slice(&digest(4)); + bytes.extend_from_slice(&digest(5)); + for budget_value in [1_u64, 2, 3] { + bytes.extend_from_slice(&budget_value.to_le_bytes()); + } + for identity_seed in 6..14 { + bytes.extend_from_slice(&digest(identity_seed)); + } + retain_action_decision_coordinate_v1( + &mut bytes, + EchoOperationActionDecisionCoordinateV1 { + writer_head: WriterHeadKey { + worldline_id: crate::WorldlineId::from_bytes(digest(14)), + head_id: crate::make_head_id("blocker-count-guard"), + }, + worldline_tick_after: WorldlineTick::from_raw(1), + commit_global_tick: GlobalTick::from_raw(1), + commit_id: digest(15), + tick_receipt_digest: digest(16), + member_index: 0, + }, + ); + bytes.extend_from_slice(&u64::MAX.to_le_bytes()); + + let error = recover_action_outcome_v1(&bytes) + .expect_err("an impossible blocker count must fail before allocation"); + assert_eq!( + error.kind(), + EchoOperationArtifactErrorKindV1::InvalidStructure + ); + assert_eq!(error.detail(), "Action blocker bytes are truncated"); + } + #[test] fn retained_installation_rejects_identity_substitution() { let installed = retained_fixture_installation(); @@ -5327,6 +6815,18 @@ mod tests { recover_committed_execution_receipt_v1(&forged_composition_bytes) .expect_err("recovery must independently derive singleton composition identity"); + let mut contextless_composite = receipt.clone(); + contextless_composite.committed_patch_digest = Some(digest(92)); + contextless_composite.composition_digest = Some(digest(93)); + contextless_composite.terminal_outcome_digest = + terminal_outcome_digest(&contextless_composite); + contextless_composite.receipt_digest = receipt_digest(&contextless_composite); + let contextless_composite_bytes = contextless_composite + .to_canonical_bytes() + .expect("contextless composite receipt encodes"); + recover_committed_execution_receipt_v1(&contextless_composite_bytes) + .expect_err("a composite receipt requires its complete scheduler batch context"); + let mut impossible_budget = receipt.clone(); impossible_budget.delegated_budget = EchoOperationBudgetV1::new(1, 1, 1); impossible_budget.receipt_digest = receipt_digest(&impossible_budget); diff --git a/crates/warp-core/src/engine_impl.rs b/crates/warp-core/src/engine_impl.rs index aea41993..fa382e88 100644 --- a/crates/warp-core/src/engine_impl.rs +++ b/crates/warp-core/src/engine_impl.rs @@ -2988,7 +2988,10 @@ fn attach_footprint_guards( Ok(()) } -fn footprints_conflict(a: &crate::footprint::Footprint, b: &crate::footprint::Footprint) -> bool { +pub(crate) fn footprints_conflict( + a: &crate::footprint::Footprint, + b: &crate::footprint::Footprint, +) -> bool { // IMPORTANT: do not use `Footprint::independent` here yet. // // This logic MUST remain consistent with the scheduler’s footprint conflict diff --git a/crates/warp-core/src/head_inbox.rs b/crates/warp-core/src/head_inbox.rs index b892b0c0..233ac05f 100644 --- a/crates/warp-core/src/head_inbox.rs +++ b/crates/warp-core/src/head_inbox.rs @@ -680,6 +680,8 @@ pub struct HeadInbox { head_key: WriterHeadKey, pending: BTreeMap, policy: InboxPolicy, + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + partitioned_filtered_removals_for_test: usize, } impl Default for HeadInbox { @@ -691,6 +693,8 @@ impl Default for HeadInbox { }, pending: BTreeMap::new(), policy: InboxPolicy::AcceptAll, + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + partitioned_filtered_removals_for_test: 0, } } } @@ -703,6 +707,8 @@ impl HeadInbox { head_key, pending: BTreeMap::new(), policy, + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + partitioned_filtered_removals_for_test: 0, } } @@ -791,6 +797,88 @@ impl HeadInbox { } } + /// Admits one deterministic execution category without mixing it with + /// other pending categories. + /// + /// When both categories are pending, `parent_global_tick` parity chooses + /// whether this batch contains the supplied `partition_kind` or everything + /// else. The durable scheduler-round coordinate advances once per scheduler + /// pass, so every participating head alternates categories even when several + /// heads advance one shared worldline. Recovery does not depend on + /// process-local state. When only one category is pending, it proceeds + /// immediately. Existing per-Tick limits still bound the selected category. + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + pub(crate) fn admit_partitioned( + &mut self, + partition_kind: IntentKind, + partition_limit: usize, + parent_global_tick: crate::GlobalTick, + ) -> Vec { + if self.pending.is_empty() { + return Vec::new(); + } + let mut has_partition = false; + let mut has_other = false; + for envelope in self.pending.values() { + let in_partition = matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ); + has_partition |= in_partition; + has_other |= !in_partition; + if has_partition && has_other { + break; + } + } + let selected_partition = match (has_partition, has_other) { + (true, true) => parent_global_tick.as_u64().is_multiple_of(2), + (true, false) => true, + (false, true) => false, + (false, false) => return Vec::new(), + }; + let policy_limit = match self.policy { + InboxPolicy::Budgeted { max_per_tick } => max_per_tick as usize, + InboxPolicy::AcceptAll | InboxPolicy::KindFilter(_) => usize::MAX, + }; + let limit = if selected_partition { + policy_limit.min(partition_limit) + } else { + policy_limit + }; + if limit == 0 { + return Vec::new(); + } + if !(has_partition && has_other) && limit >= self.pending.len() { + return std::mem::take(&mut self.pending).into_values().collect(); + } + + let mut selected_ids = Vec::new(); + for (ingress_id, envelope) in &self.pending { + let in_partition = matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ); + if in_partition == selected_partition { + selected_ids.push(*ingress_id); + if selected_ids.len() == limit { + break; + } + } + } + selected_ids + .into_iter() + .filter_map(|ingress_id| { + #[cfg(all(test, feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + { + self.partitioned_filtered_removals_for_test += 1; + } + self.pending.remove(&ingress_id) + }) + .collect() + } + /// Returns `true` if calling [`HeadInbox::admit`] would yield at least one envelope. #[must_use] pub fn can_admit(&self) -> bool { @@ -899,6 +987,258 @@ mod tests { } } + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn partitioned_admission_never_mixes_execution_categories() { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::AcceptAll, + ); + let partition_kind = test_kind(); + let envelopes = [ + make_envelope(partition_kind, b"partition-a"), + make_envelope(other_kind(), b"legacy-a"), + make_envelope(partition_kind, b"partition-b"), + make_envelope(other_kind(), b"legacy-b"), + ]; + let first_is_partition = envelopes + .iter() + .min_by_key(|envelope| envelope.ingress_id()) + .is_some_and(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ) + }); + for envelope in envelopes { + assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); + } + + let first_tick = crate::GlobalTick::from_raw(u64::from(!first_is_partition)); + let first_batch = inbox.admit_partitioned(partition_kind, 2, first_tick); + assert_eq!(first_batch.len(), 2); + assert!(first_batch.iter().all(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if (*intent_kind == partition_kind) == first_is_partition + ) + })); + assert_eq!(inbox.pending_count(), 2); + + let second_tick = first_tick + .checked_add(1) + .expect("the fixture Tick can advance"); + let second_batch = inbox.admit_partitioned(partition_kind, 2, second_tick); + assert_eq!(second_batch.len(), 2); + assert!(second_batch.iter().all(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if (*intent_kind == partition_kind) != first_is_partition + ) + })); + assert!(inbox.is_empty()); + } + + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn homogeneous_unbounded_partitioned_admission_moves_the_whole_inbox() { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::AcceptAll, + ); + let partition_kind = test_kind(); + for payload in [b"partition-a", b"partition-b", b"partition-c"] { + assert_eq!( + inbox.ingest(make_envelope(partition_kind, payload)), + InboxIngestResult::Accepted + ); + } + + let admitted = inbox.admit_partitioned(partition_kind, usize::MAX, crate::GlobalTick::ZERO); + + assert_eq!(admitted.len(), 3); + assert!(inbox.is_empty()); + assert_eq!( + inbox.partitioned_filtered_removals_for_test, 0, + "homogeneous unbounded admission must move the map without per-key removals" + ); + } + + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn scheduler_round_alternates_each_head_despite_even_worldline_progress() { + let partition_kind = test_kind(); + for head_label in ["first", "second"] { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id(head_label), + }, + InboxPolicy::Budgeted { max_per_tick: 1 }, + ); + for index in 0..2_u8 { + assert_eq!( + inbox.ingest(make_envelope( + partition_kind, + format!("{head_label}-partition-{index}").as_bytes(), + )), + InboxIngestResult::Accepted + ); + assert_eq!( + inbox.ingest(make_envelope( + other_kind(), + format!("{head_label}-other-{index}").as_bytes(), + )), + InboxIngestResult::Accepted + ); + } + + let first_batch = inbox.admit_partitioned(partition_kind, 1, crate::GlobalTick::ZERO); + let second_batch = + inbox.admit_partitioned(partition_kind, 1, crate::GlobalTick::from_raw(1)); + let is_partition = |envelope: &IngressEnvelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ) + }; + assert_eq!(first_batch.len(), 1); + assert_eq!(second_batch.len(), 1); + assert_ne!( + is_partition(&first_batch[0]), + is_partition(&second_batch[0]), + "each head must switch categories on the next scheduler round" + ); + } + } + + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn partitioned_admission_applies_both_limit_directions_and_preserves_other_partition() { + for (policy_limit, partition_limit, expected_partition_count) in + [(2_u32, 5_usize, 2_usize), (5_u32, 3_usize, 3_usize)] + { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl( + u8::try_from(policy_limit).expect("the fixture policy limit fits in u8") + ), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::Budgeted { + max_per_tick: policy_limit, + }, + ); + let partition_kind = test_kind(); + for index in 0..6_u8 { + assert_eq!( + inbox.ingest(make_envelope( + partition_kind, + format!("partition-{policy_limit}-{index}").as_bytes(), + )), + InboxIngestResult::Accepted + ); + } + let other = (0..2_u8) + .map(|index| { + make_envelope( + other_kind(), + format!("other-{policy_limit}-{index}").as_bytes(), + ) + }) + .collect::>(); + let mut other_ids = other + .iter() + .map(IngressEnvelope::ingress_id) + .collect::>(); + other_ids.sort_unstable(); + for envelope in other { + assert_eq!(inbox.ingest(envelope), InboxIngestResult::Accepted); + } + + let partition_batch = + inbox.admit_partitioned(partition_kind, partition_limit, crate::GlobalTick::ZERO); + assert_eq!(partition_batch.len(), expected_partition_count); + assert!(partition_batch.iter().all(|envelope| { + matches!( + envelope.payload(), + IngressPayload::LocalIntent { intent_kind, .. } + if *intent_kind == partition_kind + ) + })); + + let other_batch = inbox.admit_partitioned( + partition_kind, + partition_limit, + crate::GlobalTick::from_raw(1), + ); + assert_eq!( + other_batch + .iter() + .map(IngressEnvelope::ingress_id) + .collect::>(), + other_ids, + "the non-selected partition must remain byte-for-byte pending" + ); + assert_eq!(inbox.pending_count(), 6 - expected_partition_count); + } + } + + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] + #[test] + fn partitioned_admission_cannot_starve_other_category_by_ingress_hash() { + let mut inbox = HeadInbox::new( + WriterHeadKey { + worldline_id: wl(1), + head_id: crate::head::make_head_id("default"), + }, + InboxPolicy::AcceptAll, + ); + let partition_kind = test_kind(); + let legacy = (0..256_u16) + .map(|index| make_envelope(other_kind(), format!("legacy-{index}").as_bytes())) + .max_by_key(IngressEnvelope::ingress_id) + .expect("the deterministic legacy candidate set is nonempty"); + let legacy_ingress_id = legacy.ingress_id(); + assert_eq!(inbox.ingest(legacy), InboxIngestResult::Accepted); + + let mut legacy_admitted = false; + for round in 0..2_u16 { + let partition = (0..1024_u16) + .map(|index| { + make_envelope( + partition_kind, + format!("partition-{round}-{index}").as_bytes(), + ) + }) + .find(|envelope| envelope.ingress_id() < legacy_ingress_id) + .expect("the deterministic partition candidates include a lower ingress hash"); + assert_eq!(inbox.ingest(partition), InboxIngestResult::Accepted); + let batch = inbox.admit_partitioned( + partition_kind, + 1, + crate::GlobalTick::from_raw(u64::from(round)), + ); + assert_eq!(batch.len(), 1); + legacy_admitted |= batch[0].ingress_id() == legacy_ingress_id; + } + + assert!( + legacy_admitted, + "a continuously replenished lower-hash partition must not starve legacy work" + ); + } + #[test] fn re_ingesting_same_envelope_is_idempotent() { let mut inbox = HeadInbox::new( diff --git a/crates/warp-core/src/lib.rs b/crates/warp-core/src/lib.rs index 02a0ab7b..e3e058e8 100644 --- a/crates/warp-core/src/lib.rs +++ b/crates/warp-core/src/lib.rs @@ -234,28 +234,31 @@ pub use dynamic_binding::{ StructuredBindingRuntime, StructuredRuntimeBindings, }; pub use echo_operation::{ + echo_operation_action_envelope_v1, echo_operation_action_intent_kind_v1, echo_operation_anchored_node_absent_application_basis_v1, echo_operation_anchored_node_application_basis_v1, echo_operation_anchored_node_creation_application_basis_v1, echo_operation_atom_value_digest_v1, echo_operation_create_if_absent_target_profile_identity_v1, echo_operation_package_id_v1, echo_operation_target_profile_identity_v1, AdmittedEchoOperationInvocationV1, - AdmittedExecutableOperationPackageV1, EchoOperationAdmissionErrorKindV1, - EchoOperationAdmissionErrorV1, EchoOperationAdmissionPolicyV1, - EchoOperationAnchoredNodeOccupancyV1, EchoOperationApplicationBasisV1, - EchoOperationArtifactErrorKindV1, EchoOperationArtifactErrorV1, EchoOperationBudgetV1, - EchoOperationCommitErrorV1, EchoOperationEvaluationBasisIdV1, EchoOperationEvaluationBasisV1, - EchoOperationExecutionEvidenceV1, EchoOperationFootprintContractV1, - EchoOperationInstallationErrorKindV1, EchoOperationInstallationErrorV1, - EchoOperationInvocationAdmissionErrorKindV1, EchoOperationInvocationAdmissionErrorV1, - EchoOperationInvocationAdmissionIdV1, EchoOperationInvocationAdmissionPolicyV1, - EchoOperationInvocationIdV1, EchoOperationInvocationV1, EchoOperationObstructionIdV1, - EchoOperationObstructionKindV1, EchoOperationObstructionV1, EchoOperationPackageAdmissionIdV1, - EchoOperationPackageIdV1, EchoOperationPreparationV1, EchoOperationPrivateEvaluationIdV1, - EchoOperationProgramIdV1, EchoOperationProgramV1, EchoOperationReceiptV1, - EchoOperationResultIdV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, - ExecutableOperationPackageV1, InstalledEchoOperationIdV1, InstalledEchoOperationV1, - PreparedEchoOperationIdV1, PreparedEchoOperationV1, + AdmittedExecutableOperationPackageV1, EchoOperationActionDecisionCoordinateV1, + EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorKindV1, EchoOperationAdmissionErrorV1, + EchoOperationAdmissionPolicyV1, EchoOperationAnchoredNodeOccupancyV1, + EchoOperationApplicationBasisV1, EchoOperationArtifactErrorKindV1, + EchoOperationArtifactErrorV1, EchoOperationBudgetV1, EchoOperationCommitErrorV1, + EchoOperationEvaluationBasisIdV1, EchoOperationEvaluationBasisV1, + EchoOperationExecutionEvidenceV1, EchoOperationFootprintConflictV1, + EchoOperationFootprintContractV1, EchoOperationInstallationErrorKindV1, + EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorKindV1, + EchoOperationInvocationAdmissionErrorV1, EchoOperationInvocationAdmissionIdV1, + EchoOperationInvocationAdmissionPolicyV1, EchoOperationInvocationIdV1, + EchoOperationInvocationV1, EchoOperationObstructionIdV1, EchoOperationObstructionKindV1, + EchoOperationObstructionV1, EchoOperationPackageAdmissionIdV1, EchoOperationPackageIdV1, + EchoOperationPreparationV1, EchoOperationPrivateEvaluationIdV1, EchoOperationProgramIdV1, + EchoOperationProgramV1, EchoOperationReceiptV1, EchoOperationResultIdV1, + EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, ExecutableOperationPackageV1, + InstalledEchoOperationIdV1, InstalledEchoOperationV1, PreparedEchoOperationIdV1, + PreparedEchoOperationV1, ACTION_BATCH_CANDIDATE_LIMIT_V1, }; pub use edict_target_ir::{ accept_edict_echo_target_ir, execute_accepted_edict_echo_target_ir, AcceptedEdictEchoTargetIr, @@ -477,8 +480,8 @@ pub use tick_patch::{ #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] pub use trusted_runtime_host::{ EvidenceCatalogPosture, RuntimeWalActivationGap, TrustedRuntimeApp, TrustedRuntimeHost, - TrustedRuntimeHostError, TrustedRuntimeHostRunReport, TrustedRuntimeWal, - TrustedRuntimeWalConfig, TrustedRuntimeWalError, TrustedRuntimeWalRecovery, + TrustedRuntimeHostError, TrustedRuntimeHostParts, TrustedRuntimeHostRunReport, + TrustedRuntimeWal, TrustedRuntimeWalConfig, TrustedRuntimeWalError, TrustedRuntimeWalRecovery, TrustedRuntimeWalStoreKind, WitnessedCausalAnchorAdmission, }; pub use tx::TxId; diff --git a/crates/warp-core/src/provenance_codec.rs b/crates/warp-core/src/provenance_codec.rs index a14462a7..7db3aef5 100644 --- a/crates/warp-core/src/provenance_codec.rs +++ b/crates/warp-core/src/provenance_codec.rs @@ -286,6 +286,9 @@ fn push_tick_receipt(out: &mut Vec, receipt: &TickReceipt) { out.push(match entry.disposition { TickReceiptDisposition::Applied => 1, TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) => 2, + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) => 3, }); let blockers = receipt.blocked_by(index); push_len(out, blockers.len()); @@ -611,6 +614,9 @@ impl<'a> RetainedProvenanceCursor<'a> { let disposition = match self.read_u8()? { 1 => TickReceiptDisposition::Applied, 2 => TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict), + 3 => TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ), tag => return Err(unknown_tag("tick receipt disposition", tag)), }; let blocker_count = self.read_count(4)?; diff --git a/crates/warp-core/src/receipt.rs b/crates/warp-core/src/receipt.rs index c0a5ef92..030afade 100644 --- a/crates/warp-core/src/receipt.rs +++ b/crates/warp-core/src/receipt.rs @@ -87,6 +87,14 @@ pub enum TickReceiptPartsError { /// Rejected candidate missing its blocking attribution. entry: usize, }, + /// A non-conflict obstruction cannot attribute an applied blocker. + #[error("obstructed tick receipt entry {entry} carries {blocker_count} blockers")] + ObstructedEntryHasBlockers { + /// Obstructed candidate carrying invalid blocker attribution. + entry: usize, + /// Number of invalid blockers. + blocker_count: usize, + }, } impl TickReceipt { @@ -141,8 +149,19 @@ impl TickReceipt { entry: entry_index, }); } + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) if !blockers.is_empty() => { + return Err(TickReceiptPartsError::ObstructedEntryHasBlockers { + entry: entry_index, + blocker_count: blockers.len(), + }); + } TickReceiptDisposition::Applied - | TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) => {} + | TickReceiptDisposition::Rejected( + TickReceiptRejection::FootprintConflict + | TickReceiptRejection::ExecutableOperationObstruction, + ) => {} } if let Some(pair) = blockers.windows(2).find(|pair| pair[0] >= pair[1]) { return Err(TickReceiptPartsError::BlockersNotStrictlyIncreasing { @@ -285,6 +304,9 @@ impl TickReceiptDisposition { pub enum TickReceiptRejection { /// Candidate footprint conflicts with an already-accepted footprint. FootprintConflict, + /// Echo's bounded executable-operation evaluator returned a typed + /// obstruction and no candidate mutation. + ExecutableOperationObstruction, } fn compute_tick_receipt_digest(entries: &[TickReceiptEntry]) -> Hash { @@ -304,6 +326,9 @@ fn compute_tick_receipt_digest(entries: &[TickReceiptEntry]) -> Hash { let code = match entry.disposition { TickReceiptDisposition::Applied => 1u8, TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) => 2u8, + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) => 3u8, }; hasher.update(&[code]); } diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 87d16c4e..85480b56 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -21,32 +21,39 @@ use crate::{ affected_frontiers_root, build_causal_anchor_admission_transaction, build_executable_operation_installation_transaction, build_executable_operation_tick_transaction, build_recovery_certificate, - build_replayable_tick_transaction, build_submission_acceptance_with_material_transaction, + build_replayable_tick_batch_transaction, build_replayable_tick_transaction, + build_submission_acceptance_with_material_transaction, causal_anchor_frontier_digest_from_evidence, causal_anchor_genesis_frontier_digest, - causal_history_genesis_frontier_digest, logical_causal_history_frontier_digest, - recover_filesystem_store, recover_from_frames_and_commits, recover_receipt_index, - recover_submission_index, recovered_submission_receipt_index_root, + causal_history_genesis_frontier_digest, decode_tick_receipt_records, + logical_causal_history_frontier_digest, recover_filesystem_store, + recover_from_frames_and_commits, recover_receipt_index, recover_submission_index, + recovered_submission_receipt_index_root, tick_receipt_payload_is_batch, trusted_runtime_wal_digest, validate_recovered_causal_anchor_history, AffectedFrontier, AffectedFrontierKind, FilesystemWalStore, InMemoryWalStore, Lsn, PayloadCodecId, PayloadSchemaId, RecoveredCausalAnchorAdmission, RecoveredReceiptIndex, RecoveredSubmissionIndex, RecoveryAccessMode, RecoveryCertificate, RecoveryScanReport, - SubmissionAcceptanceRecord, SubmissionRetryPosture, TickReceiptRecord, WalAppendAuthority, - WalBuildError, WalCommittedTransaction, WalDecodeError, WalDurabilityMode, - WalReceiptCorrelationRecord, WalRecordKind, WalRecoveryError, WalRecoveryIndexError, - WalRuntimeStateDeltaRecord, WalSegmentId, WalStoreError, WalStorePort, - WalSubmissionEnvelopeRecord, WalTickDecision, WalTransactionBuilder, WalTransactionCommit, - WalTransactionId, WalTransactionKind, WriterEpochId, WriterEpochRequest, - TRUSTED_RUNTIME_WAL_DOMAIN, + SubmissionAcceptanceRecord, TickReceiptRecord, WalAppendAuthority, WalBuildError, + WalCommittedTransaction, WalDecodeError, WalDurabilityMode, WalReceiptCorrelationRecord, + WalRecordKind, WalRecoveryError, WalRecoveryIndexError, WalRuntimeStateDeltaRecord, + WalSegmentId, WalStoreError, WalStorePort, WalSubmissionEnvelopeRecord, WalTickDecision, + WalTransactionBuilder, WalTransactionCommit, WalTransactionId, WalTransactionKind, + WriterEpochId, WriterEpochRequest, TRUSTED_RUNTIME_WAL_DOMAIN, }, contract_host::{decode_canonical_eint, encode_canonical_eint}, echo_operation::{ - admit_invocation_v1, admit_package_v1, commit_prepared_to_state, - decode_invocation_route_v1, genesis_commit_id, install_recovered_v1, - installed_from_admitted, not_committed_basis_changed, - not_committed_evaluation_authority_mismatch, not_committed_installation_unavailable, - operation_descent_stack, prepare_operation_v1, recover_committed_execution_receipt_v1, - recover_installation_v1, retain_committed_execution_v1, retain_installation_v1, + action_admission_evidence_matches_v1, action_application_basis_matches_state_v1, + action_batch_composition_digest_from_receipts_v1, action_batch_patch_from_preparations_v1, + action_preparation_identity_matches_v1, admit_action_invocation_v1, admit_invocation_v1, + admit_package_v1, commit_prepared_to_state, commit_scheduler_action_batch_to_state_v1, + decode_invocation_route_v1, echo_operation_action_invocation_bytes_v1, + inspect_action_invocation_v1, install_recovered_v1, installed_from_admitted, + not_committed_basis_changed, not_committed_evaluation_authority_mismatch, + not_committed_installation_unavailable, operation_descent_stack, prepare_operation_v1, + reconstruct_action_evaluation_v1, reconstruct_action_preparation_v1, + recover_action_outcome_v1, recover_committed_execution_receipt_v1, recover_installation_v1, + retain_action_outcome_v1, retain_committed_execution_v1, retain_installation_v1, validate_receipt_installation_v1, EchoOperationEvaluationAuthorityV1, + SchedulerEchoOperationCandidateV1, }, provider_contract::admit_provider_contract_package_v1, AdmittedEchoOperationInvocationV1, AdmittedExecutableOperationPackageV1, @@ -54,12 +61,13 @@ use crate::{ CausalAnchorError, CausalAnchorId, CausalAnchorRootSupportPolicy, CausalAnchorSupportError, CausalFrontierRef, ContractInverseAdmissionRequest, ContractInverseContext, ContractInverseDerivation, ContractInverseHistoryObstruction, ContractInverseObstruction, - ContractOperationKind, EchoOperationAdmissionErrorV1, EchoOperationAdmissionPolicyV1, - EchoOperationApplicationBasisV1, EchoOperationArtifactErrorV1, EchoOperationCommitErrorV1, - EchoOperationEvaluationBasisV1, EchoOperationExecutionEvidenceV1, - EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorV1, - EchoOperationInvocationAdmissionPolicyV1, EchoOperationPreparationV1, EchoOperationReceiptV1, - Engine, IngressCausalParent, IngressEnvelope, IngressEnvelopeDecodeError, IngressPayload, + ContractOperationKind, EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorV1, + EchoOperationAdmissionPolicyV1, EchoOperationApplicationBasisV1, EchoOperationArtifactErrorV1, + EchoOperationCommitErrorV1, EchoOperationEvaluationBasisV1, EchoOperationExecutionEvidenceV1, + EchoOperationInstallationErrorV1, EchoOperationInvocationAdmissionErrorKindV1, + EchoOperationInvocationAdmissionErrorV1, EchoOperationInvocationAdmissionPolicyV1, + EchoOperationPreparationV1, EchoOperationReceiptV1, EchoOperationTerminalPostureV1, Engine, + IngressCausalParent, IngressEnvelope, IngressEnvelopeDecodeError, IngressPayload, IngressSubmissionGeneration, InstalledContractPackage, InstalledContractPackageError, InstalledContractPackageRecord, InstalledEchoOperationV1, InstalledProviderContractPackageRecordV1, IntentOutcome, IntentOutcomeDecision, @@ -70,7 +78,7 @@ use crate::{ ProviderContractInstallationError, ProviderContractPackageInstallerV1, ProviderContractPackageProposalV1, ProviderPackageReferenceV1, ReceiptCorrelationPersistenceRecord, ReceiptCorrelationRecord, RetainedProvenanceError, - RuntimeError, SchedulerCoordinator, StepRecord, TickReceiptRejection, + RuntimeError, SchedulerCoordinator, StepRecord, TickReceiptDisposition, TickReceiptRejection, TicketedRuntimeIngressAuthority, TicketedRuntimeIngressDisposition, WitnessedSubmissionPersistenceRecord, WitnessedSubmissionPersistenceSnapshot, WorldlineRuntime, }; @@ -83,6 +91,8 @@ const INSTALLED_CONTRACT_HOST_ADMISSION_DIGEST_DOMAIN: &[u8] = b"echo:trusted-host-installed-contract-admission:v1\0"; const PROVIDER_CONTRACT_HOST_ADMISSION_DIGEST_DOMAIN: &[u8] = b"echo:trusted-host-provider-contract-admission:v1\0"; +const ECHO_OPERATION_ACTION_ADMISSION_DIGEST_DOMAIN: &[u8] = + b"echo:trusted-host-executable-operation-action-admission:v1\0"; /// Error returned by the reference trusted host loop. #[derive(Debug, Error)] @@ -131,6 +141,12 @@ pub enum TrustedRuntimeHostError { /// Exact executable-operation retained material was malformed. #[error("trusted runtime host executable-operation artifact error: {0}")] EchoOperationArtifact(#[from] EchoOperationArtifactErrorV1), + /// Runtime-owned Action admission policy is not installed. + #[error("trusted runtime host executable-operation Action admission policy is unavailable")] + EchoOperationActionAdmissionPolicyUnavailable, + /// Runtime-owned admission refused a durably accepted executable Action. + #[error("trusted runtime host executable-operation Action admission error: {0}")] + EchoOperationActionAdmission(#[from] EchoOperationInvocationAdmissionErrorV1), /// Executable-operation installation conflicted with installed authority. #[error("trusted runtime host executable-operation installation error: {0}")] EchoOperationInstallation(#[from] EchoOperationInstallationErrorV1), @@ -329,6 +345,9 @@ pub enum TrustedRuntimeWalError { /// Number of transactions in the attempted durable batch. transaction_count: usize, }, + /// Per-Action receipt records do not describe one exact scheduler Tick. + #[error("trusted runtime WAL scheduler Tick batch is internally inconsistent")] + SchedulerTickBatchMismatch, } /// Live runtime authority category that a WAL activation cannot safely recover. @@ -361,6 +380,13 @@ pub struct TrustedRuntimeHostRunReport { pub committed_steps: usize, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct EchoOperationActionSchedulerPostureV1 { + pending: usize, + admitted: usize, + admission_obstructions: usize, +} + /// One Echo causal-anchor admission observed in committed control history. /// /// The enclosed fact and receipt are the semantic evidence. WAL coordinates on @@ -408,6 +434,20 @@ impl WitnessedCausalAnchorAdmission { } } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct RecoveredEchoOperationActionInstallationOrder { + package_ordinals: BTreeMap, + installation_count_before_tick: BTreeMap, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct RecoveredEchoOperationActionOutcomeValidationMetrics { + #[cfg(any(test, feature = "host_test"))] + indexed_package_lookups: usize, + #[cfg(any(test, feature = "host_test"))] + installation_order_lookups: usize, +} + /// Read-only runtime WAL recovery report. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TrustedRuntimeWalRecovery { @@ -433,10 +473,164 @@ pub struct TrustedRuntimeWalRecovery { pub installed_echo_operations: Vec, /// Typed receipts reconstructed from executable-operation tick records. pub echo_operation_receipts: Vec, + /// Typed executable-operation Action outcomes reconstructed from + /// scheduler-owned Tick records, keyed by witnessed submission. + pub echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, + echo_operation_action_decisions: BTreeMap, + echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder, causal_history_frontiers: Vec, } impl TrustedRuntimeWalRecovery { + /// Re-runs executable Action/Tick correspondence checks after a targeted + /// test mutates recovered evidence. + #[cfg(any(test, feature = "host_test"))] + pub fn validate_echo_operation_action_outcomes_for_test( + &self, + ) -> Result<(), TrustedRuntimeWalError> { + validate_recovered_echo_operation_action_outcomes( + &self.witnessed_submissions, + &self.receipt_correlations, + &self.provenance_entries, + &self.installed_echo_operations, + &self.echo_operation_action_outcomes, + &self.echo_operation_action_decisions, + &self.echo_operation_action_installation_order, + ) + .map(|_| ()) + } + + /// Returns the indexed package and installation-order lookup counts used + /// while validating recovered Action outcomes. + #[cfg(any(test, feature = "host_test"))] + pub fn echo_operation_action_installation_lookup_counts_for_test( + &self, + ) -> Result<(usize, usize), TrustedRuntimeWalError> { + validate_recovered_echo_operation_action_outcomes( + &self.witnessed_submissions, + &self.receipt_correlations, + &self.provenance_entries, + &self.installed_echo_operations, + &self.echo_operation_action_outcomes, + &self.echo_operation_action_decisions, + &self.echo_operation_action_installation_order, + ) + .map(|metrics| { + ( + metrics.indexed_package_lookups, + metrics.installation_order_lookups, + ) + }) + } + + /// Replaces one recovered scheduler decision for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_decision_for_test( + &mut self, + submission_id: Hash, + decision: WalTickDecision, + ) { + self.echo_operation_action_decisions + .insert(submission_id, decision); + } + + /// Removes the installation-order witness for one Action in adversarial tests. + #[cfg(any(test, feature = "host_test"))] + pub fn clear_echo_operation_action_installations_before_tick_for_test( + &mut self, + submission_id: Hash, + ) { + self.echo_operation_action_installation_order + .installation_count_before_tick + .remove(&submission_id); + } + + /// Replaces one retained obstruction kind for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_obstruction_kind_for_test( + &mut self, + submission_id: Hash, + kind: crate::EchoOperationObstructionKindV1, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(candidate, _, _)| *candidate == submission_id) + { + outcome.replace_obstruction_kind_for_test(kind); + } + } + + /// Replaces one rejected preparation identity for adversarial tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_conflict_preparation_for_test( + &mut self, + submission_id: Hash, + digest: Hash, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(candidate, _, _)| *candidate == submission_id) + { + outcome.replace_conflict_preparation_id_for_test(digest); + } + } + + /// Replaces one retained conflict's blockers for adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_conflict_blockers_for_test( + &mut self, + submission_id: Hash, + replacement: Vec, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(retained_submission_id, _, _)| *retained_submission_id == submission_id) + { + outcome.replace_conflict_blockers_for_test(replacement); + } + } + + /// Replaces one noncommitted Action outcome's deciding member index for + /// adversarial recovery tests. + #[cfg(any(test, feature = "host_test"))] + pub fn replace_echo_operation_action_decision_member_index_for_test( + &mut self, + submission_id: Hash, + replacement: u32, + ) { + if let Some((_, _, outcome)) = self + .echo_operation_action_outcomes + .iter_mut() + .find(|(retained_submission_id, _, _)| *retained_submission_id == submission_id) + { + outcome.replace_decision_member_index_for_test(replacement); + } + } + + /// Re-runs activation-time Action parent-state checks for adversarial tests. + #[cfg(any(test, feature = "host_test"))] + pub fn validate_echo_operation_parent_states_for_test( + &self, + runtime: &WorldlineRuntime, + provenance: &ProvenanceService, + ) -> Result<(), TrustedRuntimeWalError> { + validate_recovered_echo_operation_parent_states(runtime, provenance, self).map(|_| ()) + } + + /// Returns the number of causal-state reconstructions used by Action + /// parent-state validation. + #[cfg(any(test, feature = "host_test"))] + pub fn echo_operation_parent_state_replay_count_for_test( + &self, + runtime: &WorldlineRuntime, + provenance: &ProvenanceService, + ) -> Result { + validate_recovered_echo_operation_parent_states(runtime, provenance, self) + } + /// Recomputes the certificate's canonical index root from recovered evidence. /// /// # Errors @@ -454,6 +648,7 @@ impl TrustedRuntimeWalRecovery { causal_anchor_history: &self.causal_anchor_history, installed_echo_operations: &self.installed_echo_operations, echo_operation_receipts: &self.echo_operation_receipts, + echo_operation_action_outcomes: &self.echo_operation_action_outcomes, }) } @@ -606,6 +801,87 @@ pub struct TrustedRuntimeHost { runtime_wal: Option, causal_anchor_support_policy: Option, echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1, + echo_operation_action_admission_policy: Option, + echo_operation_action_outcomes: BTreeMap, + pending_echo_operation_actions: BTreeSet, + admitted_echo_operation_actions: BTreeMap, + echo_operation_action_admission_obstructions: + BTreeMap, +} + +/// Owned decomposition of a [`TrustedRuntimeHost`]. +/// +/// Besides the runtime, provenance service, and engine, this value retains +/// host-owned WAL, authority, policy, pending-admission, and typed Action +/// outcome state. Passing it back to [`TrustedRuntimeHost::from_parts`] is an +/// identity-preserving host lifecycle operation. +pub struct TrustedRuntimeHostParts { + runtime: WorldlineRuntime, + provenance: ProvenanceService, + engine: Engine, + runtime_wal: Option, + causal_anchor_support_policy: Option, + echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1, + echo_operation_action_admission_policy: Option, + echo_operation_action_outcomes: BTreeMap, + pending_echo_operation_actions: BTreeSet, + admitted_echo_operation_actions: BTreeMap, + echo_operation_action_admission_obstructions: + BTreeMap, +} + +impl TrustedRuntimeHostParts { + /// Returns the owned runtime as read-only evidence. + #[must_use] + pub const fn runtime(&self) -> &WorldlineRuntime { + &self.runtime + } + + /// Returns the owned runtime for host-level reconfiguration. + pub const fn runtime_mut(&mut self) -> &mut WorldlineRuntime { + &mut self.runtime + } + + /// Returns the owned provenance service as read-only evidence. + #[must_use] + pub const fn provenance(&self) -> &ProvenanceService { + &self.provenance + } + + /// Returns the owned provenance service for host-level reconfiguration. + pub const fn provenance_mut(&mut self) -> &mut ProvenanceService { + &mut self.provenance + } + + /// Returns the owned execution engine as read-only evidence. + #[must_use] + pub const fn engine(&self) -> &Engine { + &self.engine + } + + /// Returns the owned execution engine for host-level reconfiguration. + pub const fn engine_mut(&mut self) -> &mut Engine { + &mut self.engine + } +} + +fn pending_echo_operation_action_ids_v1( + runtime: &WorldlineRuntime, + decided: &BTreeMap, +) -> BTreeSet { + runtime + .pending_witnessed_submissions() + .filter_map(|submission| { + let submission_id = submission.submission_id; + (!decided.contains_key(&submission_id) + && runtime + .witnessed_submission_envelope(&submission_id) + .is_some_and(|envelope| { + echo_operation_action_invocation_bytes_v1(envelope).is_some() + })) + .then_some(submission_id) + }) + .collect() } impl TrustedRuntimeHost { @@ -617,6 +893,8 @@ impl TrustedRuntimeHost { /// Returns a provenance error if any runtime worldline cannot be registered. pub fn new(runtime: WorldlineRuntime, engine: Engine) -> Result { let provenance = provenance_from_runtime(&runtime)?; + let pending_echo_operation_actions = + pending_echo_operation_action_ids_v1(&runtime, &BTreeMap::new()); Ok(Self { runtime, provenance, @@ -624,30 +902,62 @@ impl TrustedRuntimeHost { runtime_wal: None, causal_anchor_support_policy: None, echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), + echo_operation_action_admission_policy: None, + echo_operation_action_outcomes: BTreeMap::new(), + pending_echo_operation_actions, + admitted_echo_operation_actions: BTreeMap::new(), + echo_operation_action_admission_obstructions: BTreeMap::new(), }) } - /// Builds a trusted host from already-initialized parts. + /// Rebuilds a trusted host from its identity-preserving owned parts. #[must_use] - pub fn from_parts( - runtime: WorldlineRuntime, - provenance: ProvenanceService, - engine: Engine, - ) -> Self { + pub fn from_parts(parts: TrustedRuntimeHostParts) -> Self { + let TrustedRuntimeHostParts { + runtime, + provenance, + engine, + runtime_wal, + causal_anchor_support_policy, + echo_operation_evaluation_authority, + echo_operation_action_admission_policy, + echo_operation_action_outcomes, + pending_echo_operation_actions, + admitted_echo_operation_actions, + echo_operation_action_admission_obstructions, + } = parts; Self { runtime, provenance, engine, - runtime_wal: None, - causal_anchor_support_policy: None, - echo_operation_evaluation_authority: EchoOperationEvaluationAuthorityV1::new(), + runtime_wal, + causal_anchor_support_policy, + echo_operation_evaluation_authority, + echo_operation_action_admission_policy, + echo_operation_action_outcomes, + pending_echo_operation_actions, + admitted_echo_operation_actions, + echo_operation_action_admission_obstructions, } } /// Consumes the host and returns owned runtime parts. #[must_use] - pub fn into_parts(self) -> (WorldlineRuntime, ProvenanceService, Engine) { - (self.runtime, self.provenance, self.engine) + pub fn into_parts(self) -> TrustedRuntimeHostParts { + TrustedRuntimeHostParts { + runtime: self.runtime, + provenance: self.provenance, + engine: self.engine, + runtime_wal: self.runtime_wal, + causal_anchor_support_policy: self.causal_anchor_support_policy, + echo_operation_evaluation_authority: self.echo_operation_evaluation_authority, + echo_operation_action_admission_policy: self.echo_operation_action_admission_policy, + echo_operation_action_outcomes: self.echo_operation_action_outcomes, + pending_echo_operation_actions: self.pending_echo_operation_actions, + admitted_echo_operation_actions: self.admitted_echo_operation_actions, + echo_operation_action_admission_obstructions: self + .echo_operation_action_admission_obstructions, + } } /// Returns the host-owned runtime as read-only evidence. @@ -668,6 +978,38 @@ impl TrustedRuntimeHost { &self.engine } + /// Installs the runtime-owned policy used to admit durably accepted + /// executable-operation Actions before scheduler selection. + pub fn install_echo_operation_action_admission_policy_v1( + &mut self, + policy: EchoOperationInvocationAdmissionPolicyV1, + ) { + self.echo_operation_action_admission_policy = Some(policy); + self.requeue_obstructed_echo_operation_actions_v1(); + } + + /// Returns the typed scheduler-owned outcome for one executable-operation + /// Action submission. + #[must_use] + pub fn echo_operation_action_outcome_v1( + &self, + submission_id: &Hash, + ) -> Option<&EchoOperationActionOutcomeV1> { + self.echo_operation_action_outcomes.get(submission_id) + } + + /// Returns the typed pending posture for an executable-operation Action + /// which runtime-owned admission cannot currently authorize. + #[must_use] + pub fn echo_operation_action_admission_obstruction_v1( + &self, + submission_id: &Hash, + ) -> Option { + self.echo_operation_action_admission_obstructions + .get(submission_id) + .copied() + } + /// Admits exact executable-operation package bytes under independent policy. /// /// A successful result is an opaque admission token. It does not install @@ -713,6 +1055,7 @@ impl TrustedRuntimeHost { } self.engine .restore_recovered_echo_operation_packages_v1(core::slice::from_ref(&installed))?; + self.requeue_obstructed_echo_operation_actions_v1(); Ok(installed) } @@ -726,12 +1069,13 @@ impl TrustedRuntimeHost { writer_head: crate::WriterHeadKey, application_basis: EchoOperationApplicationBasisV1, ) -> Result { - current_echo_operation_basis( + crate::coordinator::resolve_echo_operation_evaluation_basis_v1( &self.runtime, &self.provenance, writer_head, application_basis, ) + .map_err(TrustedRuntimeHostError::from) } /// Independently admits a canonical installed-operation invocation. @@ -771,11 +1115,14 @@ impl TrustedRuntimeHost { ) } - /// Evaluates one admitted operation without mutating the parent state. + /// Transitional host-only seam for evaluating one admitted operation + /// without mutating the parent state. /// - /// Evaluation returns either one complete prepared patch or one typed - /// obstruction. No partial mutation reaches the parent worldline. + /// Application execution must submit an executable-operation Action and + /// let the scheduler invoke private evaluation while constructing a Tick. + /// This direct seam remains hidden for compatibility and focused tests. #[must_use] + #[doc(hidden)] pub fn prepare_echo_operation_v1( &self, admitted: AdmittedEchoOperationInvocationV1, @@ -806,11 +1153,13 @@ impl TrustedRuntimeHost { ) } - /// Commits one privately prepared patch only against its exact evaluation basis. + /// Transitional host-only seam for directly committing one privately + /// prepared consequence against its exact evaluation basis. /// - /// A changed basis produces typed noncommit evidence and never retries, - /// rebases, revalidates, or exposes a partial patch. A successful crossing - /// appends one local provenance entry and advances one worldline tick. + /// Application execution must use scheduler-owned executable-operation + /// Actions. This direct seam remains hidden for compatibility and focused + /// tests while that older lifecycle is retired. + #[doc(hidden)] pub fn commit_prepared_echo_operation_v1( &mut self, prepared: Box, @@ -991,8 +1340,19 @@ impl TrustedRuntimeHost { self.engine .restore_recovered_echo_operation_packages_v1(&recovery.installed_echo_operations)?; + self.echo_operation_action_outcomes = recovery + .echo_operation_action_outcomes + .iter() + .map(|(submission_id, _, outcome)| (*submission_id, outcome.clone())) + .collect(); self.runtime = restored_runtime; self.provenance = restored_provenance; + self.pending_echo_operation_actions = pending_echo_operation_action_ids_v1( + &self.runtime, + &self.echo_operation_action_outcomes, + ); + self.admitted_echo_operation_actions.clear(); + self.echo_operation_action_admission_obstructions.clear(); self.runtime_wal = Some(runtime_wal); Ok(()) } @@ -1034,6 +1394,18 @@ impl TrustedRuntimeHost { Ok(()) } + /// Injects one scheduler Action Tick construction failure for a targeted + /// rollback test. + #[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") + ))] + pub fn inject_echo_operation_action_tick_construction_failure_for_test(&mut self) { + self.runtime + .inject_echo_operation_action_tick_construction_failure_for_test(); + } + /// Returns the app-facing surface. This surface can submit and observe, but /// it cannot tick, stage ticketed ingress, register packages, or recover /// scheduler faults. @@ -1489,34 +1861,182 @@ impl TrustedRuntimeHost { ) } + fn track_pending_echo_operation_action_v1( + &mut self, + submission_id: Hash, + is_echo_operation_action: bool, + ) { + if is_echo_operation_action + && !self + .echo_operation_action_outcomes + .contains_key(&submission_id) + { + self.pending_echo_operation_actions.insert(submission_id); + } + } + + fn echo_operation_action_scheduler_posture_v1(&self) -> EchoOperationActionSchedulerPostureV1 { + EchoOperationActionSchedulerPostureV1 { + pending: self.pending_echo_operation_actions.len(), + admitted: self.admitted_echo_operation_actions.len(), + admission_obstructions: self.echo_operation_action_admission_obstructions.len(), + } + } + + fn requeue_obstructed_echo_operation_actions_v1(&mut self) { + self.pending_echo_operation_actions.extend( + self.echo_operation_action_admission_obstructions + .keys() + .copied(), + ); + self.echo_operation_action_admission_obstructions.clear(); + } + + fn admit_pending_echo_operation_actions_v1( + &mut self, + ) -> Result, TrustedRuntimeHostError> { + if self.pending_echo_operation_actions.is_empty() { + return Ok(self.admitted_echo_operation_actions.clone()); + } + let Some(policy) = self.echo_operation_action_admission_policy else { + return Ok(self.admitted_echo_operation_actions.clone()); + }; + let mut available_by_head = SchedulerCoordinator::peek_order(&self.runtime) + .into_iter() + .map(|head_key| { + ( + head_key, + crate::echo_operation::ACTION_BATCH_CANDIDATE_LIMIT_V1, + ) + }) + .collect::>(); + for submission_id in self.admitted_echo_operation_actions.keys() { + let submission = self + .runtime + .witnessed_submission(submission_id) + .ok_or(RuntimeError::UnknownIntentSubmission(*submission_id))?; + if let Some(available) = available_by_head.get_mut(&submission.head_key) { + *available = available.saturating_sub(1); + } + } + let mut pending = self + .pending_echo_operation_actions + .iter() + .filter(|submission_id| { + !self + .admitted_echo_operation_actions + .contains_key(*submission_id) + }) + .map(|submission_id| { + let submission = self + .runtime + .witnessed_submission(submission_id) + .ok_or(RuntimeError::UnknownIntentSubmission(*submission_id))?; + Ok((submission.ingress_id, *submission_id, submission.head_key)) + }) + .collect::, RuntimeError>>()?; + pending.sort_unstable(); + let pending = pending + .into_iter() + .filter_map(|(ingress_id, submission_id, head_key)| { + let available = available_by_head.get_mut(&head_key)?; + if *available == 0 { + return None; + } + *available -= 1; + Some((ingress_id, submission_id)) + }) + .collect::>(); + for (_, submission_id) in pending { + let submission = self + .runtime + .witnessed_submission(&submission_id) + .cloned() + .ok_or(RuntimeError::UnknownIntentSubmission(submission_id))?; + let envelope = self + .runtime + .witnessed_submission_envelope(&submission_id) + .cloned() + .ok_or(RuntimeError::UnknownIntentSubmission(submission_id))?; + let invocation_bytes = echo_operation_action_invocation_bytes_v1(&envelope) + .ok_or(RuntimeError::UnknownIntentSubmission(submission_id))? + .to_vec(); + if let Some(runtime_wal) = self.runtime_wal.as_ref() { + let durably_accepted = runtime_wal + .has_submission_acceptance(submission.submission_id, submission.ingress_id); + if !durably_accepted { + continue; + } + } + let package_id = match decode_invocation_route_v1(&invocation_bytes) { + Ok((package_id, _)) => package_id, + Err(error) => { + self.echo_operation_action_admission_obstructions + .insert(submission.submission_id, error.kind()); + self.pending_echo_operation_actions + .remove(&submission.submission_id); + continue; + } + }; + let admitted = match admit_action_invocation_v1( + self.engine.installed_echo_operation_package_v1(package_id), + policy, + &invocation_bytes, + self.echo_operation_evaluation_authority.clone(), + ) { + Ok(admitted) => admitted, + Err(error) => { + self.echo_operation_action_admission_obstructions + .insert(submission.submission_id, error.kind()); + self.pending_echo_operation_actions + .remove(&submission.submission_id); + continue; + } + }; + let admission_digest = echo_operation_action_admission_digest(&submission, &admitted); + self.runtime.ingest_echo_operation_action_v1( + &TicketedRuntimeIngressAuthority::assume_runtime_owner(), + submission.submission_id, + admission_digest, + envelope, + )?; + self.echo_operation_action_admission_obstructions + .remove(&submission.submission_id); + self.admitted_echo_operation_actions + .insert(submission.submission_id, admitted); + } + Ok(self.admitted_echo_operation_actions.clone()) + } + /// Runs one scheduler-owned pass. /// /// # Errors /// /// Returns a runtime error if the scheduler pass fails. pub fn tick_once(&mut self) -> Result, TrustedRuntimeHostError> { - let existing_correlations = self - .runtime - .receipt_correlations() - .map(|correlation| correlation.ticketed_ingress_id) - .collect::>(); let runtime_before = self.runtime.clone(); let provenance_before = self.provenance.clone(); - let records = SchedulerCoordinator::super_tick( - &mut self.runtime, - &mut self.provenance, - &mut self.engine, - )?; + let action_outcomes_before = self.echo_operation_action_outcomes.clone(); + let admitted_actions_before = self.admitted_echo_operation_actions.clone(); + let admitted_actions = match self.admit_pending_echo_operation_actions_v1() { + Ok(admitted) => admitted, + Err(error) => { + self.runtime = runtime_before; + self.provenance = provenance_before; + self.admitted_echo_operation_actions = admitted_actions_before; + return Err(error); + } + }; + let (records, action_outcomes, new_correlations) = + SchedulerCoordinator::super_tick_with_echo_operation_actions_v1( + &mut self.runtime, + &mut self.provenance, + &mut self.engine, + &admitted_actions, + &self.echo_operation_evaluation_authority, + )?; let mut tick_wal_records = Vec::new(); if self.runtime_wal.is_some() { - let new_correlations = self - .runtime - .receipt_correlations() - .filter(|correlation| { - !existing_correlations.contains(&correlation.ticketed_ingress_id) - }) - .cloned() - .collect::>(); for correlation in new_correlations { let decision = match wal_tick_decision_from_observation( self.runtime @@ -1527,6 +2047,8 @@ impl TrustedRuntimeHost { Err(error) => { self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(error.into()); } }; @@ -1536,31 +2058,74 @@ impl TrustedRuntimeHost { Err(error) => { self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(error); } }; tick_wal_records.push((correlation, decision, state_delta, state_delta_digest)); } } + let action_outcomes_by_submission = action_outcomes + .iter() + .map(|(submission_id, outcome)| (*submission_id, outcome)) + .collect::>(); + let mut tick_wal_groups = BTreeMap::new(); + for (correlation, decision, state_delta, state_delta_digest) in tick_wal_records { + let key = ( + correlation.commit_hash, + correlation.tick_receipt_digest, + state_delta_digest, + ); + tick_wal_groups.entry(key).or_insert_with(Vec::new).push(( + correlation, + decision, + state_delta, + state_delta_digest, + )); + } if let Some(runtime_wal) = self.runtime_wal.as_mut() { - if runtime_wal.uses_filesystem_store() && tick_wal_records.len() > 1 { + if runtime_wal.uses_filesystem_store() && tick_wal_groups.len() > 1 { self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(TrustedRuntimeWalError::FilesystemAtomicBatchUnsupported { transaction_kind: WalTransactionKind::SchedulerTick, - transaction_count: tick_wal_records.len(), + transaction_count: tick_wal_groups.len(), } .into()); } let runtime_wal_before = runtime_wal.clone(); - for (correlation, decision, state_delta, state_delta_digest) in &tick_wal_records { - if let Err(error) = runtime_wal.record_tick_receipt( - correlation, - *decision, - state_delta, - *state_delta_digest, - ) { - if runtime_wal.recover_filesystem_tick_commit_after_error(correlation) { + for group in tick_wal_groups.values() { + let Some((first_correlation, _, state_delta, state_delta_digest)) = group.first() + else { + continue; + }; + let group_has_action_outcomes = group.iter().any(|(correlation, _, _, _)| { + action_outcomes_by_submission.contains_key(&correlation.submission_id) + }); + let result = if group.len() == 1 && !group_has_action_outcomes { + runtime_wal.record_tick_receipt( + first_correlation, + group[0].1, + state_delta, + *state_delta_digest, + ) + } else { + let correlations = group + .iter() + .map(|(correlation, decision, _, _)| (correlation.clone(), *decision)) + .collect::>(); + runtime_wal.record_tick_receipt_batch( + &correlations, + &action_outcomes_by_submission, + state_delta, + *state_delta_digest, + ) + }; + if let Err(error) = result { + if runtime_wal.recover_filesystem_tick_commit_after_error(first_correlation) { continue; } if !runtime_wal.uses_filesystem_store() { @@ -1568,14 +2133,25 @@ impl TrustedRuntimeHost { } self.runtime = runtime_before; self.provenance = provenance_before; + self.echo_operation_action_outcomes = action_outcomes_before; + self.admitted_echo_operation_actions = admitted_actions_before; return Err(error.into()); } } } + for (submission_id, outcome) in action_outcomes { + self.pending_echo_operation_actions.remove(&submission_id); + self.admitted_echo_operation_actions.remove(&submission_id); + self.echo_operation_action_admission_obstructions + .remove(&submission_id); + self.echo_operation_action_outcomes + .insert(submission_id, outcome); + } Ok(records) } - /// Runs scheduler-owned passes until an idle pass occurs. + /// Runs scheduler-owned passes until one pass commits no steps and makes + /// no executable-Action admission progress. /// /// # Errors /// @@ -1598,9 +2174,12 @@ impl TrustedRuntimeHost { max_scheduler_passes, }); } + let action_posture_before = self.echo_operation_action_scheduler_posture_v1(); let steps = self.tick_once()?; + let action_admission_progressed = + self.echo_operation_action_scheduler_posture_v1() != action_posture_before; report.scheduler_passes += 1; - if steps.is_empty() { + if steps.is_empty() && !action_admission_progressed { return Ok(report); } report.committed_steps += steps.len(); @@ -1624,37 +2203,6 @@ impl ProviderContractPackageInstallerV1 for TrustedRuntimeHost { } } -fn current_echo_operation_basis( - runtime: &WorldlineRuntime, - provenance: &ProvenanceService, - writer_head: crate::WriterHeadKey, - application_basis: EchoOperationApplicationBasisV1, -) -> Result { - if runtime.heads().get(&writer_head).is_none() { - return Err(RuntimeError::UnknownHead(writer_head).into()); - } - let frontier = runtime - .worldlines() - .get(&writer_head.worldline_id) - .ok_or(RuntimeError::UnknownWorldline(writer_head.worldline_id))?; - let state_root = frontier.state().state_root(); - let (commit_global_tick, commit_id) = match provenance.tip_ref(writer_head.worldline_id)? { - Some(tip) => { - let entry = provenance.entry(tip.worldline_id, tip.worldline_tick)?; - (Some(entry.commit_global_tick), entry.expected.commit_hash) - } - None => (None, genesis_commit_id(writer_head, state_root)), - }; - Ok(EchoOperationEvaluationBasisV1::new( - writer_head, - frontier.frontier_tick(), - commit_global_tick, - state_root, - commit_id, - application_basis, - )) -} - fn ensure_runtime_authority_is_durable( runtime: &WorldlineRuntime, provenance: &ProvenanceService, @@ -1875,7 +2423,7 @@ pub struct TrustedRuntimeWal { evidence_catalog_posture: EvidenceCatalogPosture, #[cfg(any(test, feature = "host_test"))] fail_next_evidence_catalog_update: bool, - #[cfg(test)] + #[cfg(any(test, feature = "host_test"))] recover_read_only_call_count: std::cell::Cell, writer_epoch: WriterEpochId, segment_id: WalSegmentId, @@ -1894,6 +2442,7 @@ pub struct TrustedRuntimeWal { causal_anchor_frontier_digest: Hash, causal_history_frontier_digest: Hash, causal_anchor_claim_projection: CausalAnchorClaimProjection, + durable_submission_acceptances: BTreeMap, } impl TrustedRuntimeWal { @@ -1913,6 +2462,13 @@ impl TrustedRuntimeWal { let mut store = TrustedRuntimeWalStore::open(store)?; let recovery_report = store.recover_for_writer()?; let recovered_cursor = TrustedRuntimeWalCursor::from_recovery(&recovery_report)?; + let durable_submission_acceptances = recover_submission_index(&recovery_report) + .map_err(WalRecoveryError::from)? + .entries() + .map(|(submission_id, entry)| { + (*submission_id, entry.acceptance.canonical_envelope_digest) + }) + .collect(); let evidence_catalog = crate::evidence::CausalSegmentCatalog::from_recovery_scan(&recovery_report)?; let next_lsn = if recovered_cursor.has_committed_history { @@ -1958,11 +2514,12 @@ impl TrustedRuntimeWal { causal_anchor_frontier_digest: recovered_cursor.causal_anchor_frontier_digest, causal_history_frontier_digest: recovered_cursor.causal_history_frontier_digest, causal_anchor_claim_projection: recovered_cursor.causal_anchor_claim_projection, + durable_submission_acceptances, evidence_catalog: Some(evidence_catalog), evidence_catalog_posture: EvidenceCatalogPosture::Fresh, #[cfg(any(test, feature = "host_test"))] fail_next_evidence_catalog_update: false, - #[cfg(test)] + #[cfg(any(test, feature = "host_test"))] recover_read_only_call_count: std::cell::Cell::new(0), }) } @@ -1992,10 +2549,30 @@ impl TrustedRuntimeWal { self.store.cloned_in_memory_store() } + /// Re-runs state-delta recovery after repeating the last scheduler + /// transaction, for adversarial transaction-atomicity tests. + #[cfg(any(test, feature = "host_test"))] + pub fn recover_with_repeated_scheduler_tick_for_test( + &self, + ) -> Result<(), TrustedRuntimeWalError> { + let mut report = self.store.recover_read_only()?; + let repeated = report + .transactions + .iter() + .rev() + .find(|transaction| { + transaction.commit.transaction_kind == WalTransactionKind::SchedulerTick + }) + .cloned() + .ok_or_else(missing_trusted_runtime_record)?; + report.transactions.push(repeated); + recover_runtime_state_delta_material(&report).map(|_| ()) + } + /// Recovers submission and receipt indexes from committed WAL transactions /// without scheduler callbacks. pub fn recover_read_only(&self) -> Result { - #[cfg(test)] + #[cfg(any(test, feature = "host_test"))] self.recover_read_only_call_count .set(self.recover_read_only_call_count.get() + 1); let report = self.store.recover_read_only()?; @@ -2010,10 +2587,23 @@ impl TrustedRuntimeWal { recover_echo_operation_material(&report, &runtime_state.provenance_entries)?; let provenance_entries = runtime_state.provenance_entries; let receipt_correlations = runtime_state.receipt_correlations; + let echo_operation_action_outcomes = runtime_state.echo_operation_action_outcomes; + let echo_operation_action_decisions = runtime_state.echo_operation_action_decisions; + let echo_operation_action_installation_order = + runtime_state.echo_operation_action_installation_order; let missing_runtime_state_deltas = runtime_state.missing_runtime_state_deltas; let installed_echo_operations = operation_material.installations; let echo_operation_receipts = operation_material.receipts; validate_recovered_causal_parent_evidence(&witnessed_submissions, &receipt_correlations)?; + validate_recovered_echo_operation_action_outcomes( + &witnessed_submissions, + &receipt_correlations, + &provenance_entries, + &installed_echo_operations, + &echo_operation_action_outcomes, + &echo_operation_action_decisions, + &echo_operation_action_installation_order, + )?; let certificate = runtime_wal_recovery_certificate( &report, &RecoveredRuntimeWalIndexEvidence { @@ -2026,6 +2616,7 @@ impl TrustedRuntimeWal { causal_anchor_history: &causal_anchor_history, installed_echo_operations: &installed_echo_operations, echo_operation_receipts: &echo_operation_receipts, + echo_operation_action_outcomes: &echo_operation_action_outcomes, }, )?; Ok(TrustedRuntimeWalRecovery { @@ -2040,10 +2631,27 @@ impl TrustedRuntimeWal { causal_anchor_history, installed_echo_operations, echo_operation_receipts, + echo_operation_action_outcomes, + echo_operation_action_decisions, + echo_operation_action_installation_order, causal_history_frontiers, }) } + /// Resets test instrumentation for full read-only WAL recovery. + #[cfg(any(test, feature = "host_test"))] + pub fn reset_recover_read_only_call_count_for_test(&self) { + self.recover_read_only_call_count.set(0); + } + + /// Returns the number of full read-only WAL recoveries since the last + /// instrumentation reset. + #[cfg(any(test, feature = "host_test"))] + #[must_use] + pub fn recover_read_only_call_count_for_test(&self) -> usize { + self.recover_read_only_call_count.get() + } + /// Recovers the causal segment catalog from committed WAL transactions. pub fn recover_evidence_catalog_read_only( &self, @@ -2123,17 +2731,8 @@ impl TrustedRuntimeWal { &self, submission_id: Hash, canonical_envelope_digest: Hash, - ) -> Result { - let recovery = self.recover_read_only()?; - Ok(matches!( - recovery - .submissions - .retry_posture(submission_id, canonical_envelope_digest), - SubmissionRetryPosture::AlreadyAcceptedPending - | SubmissionRetryPosture::AlreadyDecidedApplied - | SubmissionRetryPosture::AlreadyDecidedRejected - | SubmissionRetryPosture::AlreadyObstructed - )) + ) -> bool { + self.durable_submission_acceptances.get(&submission_id) == Some(&canonical_envelope_digest) } fn uses_filesystem_store(&self) -> bool { @@ -2153,6 +2752,13 @@ impl TrustedRuntimeWal { fn refresh_cursor_from_store_for_writer(&mut self) -> Result<(), TrustedRuntimeWalError> { let report = self.store.recover_for_writer()?; let cursor = TrustedRuntimeWalCursor::from_recovery(&report)?; + let durable_submission_acceptances = recover_submission_index(&report) + .map_err(WalRecoveryError::from)? + .entries() + .map(|(submission_id, entry)| { + (*submission_id, entry.acceptance.canonical_envelope_digest) + }) + .collect(); match crate::evidence::CausalSegmentCatalog::from_recovery_scan(&report) { Ok(catalog) => { self.evidence_catalog = Some(catalog); @@ -2178,6 +2784,7 @@ impl TrustedRuntimeWal { self.causal_anchor_frontier_digest = cursor.causal_anchor_frontier_digest; self.causal_history_frontier_digest = cursor.causal_history_frontier_digest; self.causal_anchor_claim_projection = cursor.causal_anchor_claim_projection; + self.durable_submission_acceptances = durable_submission_acceptances; Ok(()) } @@ -2193,7 +2800,6 @@ impl TrustedRuntimeWal { return false; } self.has_submission_acceptance(submission_id, canonical_envelope_digest) - .unwrap_or(false) } fn recover_filesystem_tick_commit_after_error( @@ -2297,6 +2903,8 @@ impl TrustedRuntimeWal { )?; let commit = self.append_transaction(transaction)?; self.submission_frontier_digest = next_submission_frontier; + self.durable_submission_acceptances + .insert(record.submission_id, record.canonical_envelope_digest); Ok(commit) } @@ -2354,6 +2962,107 @@ impl TrustedRuntimeWal { Ok(commit) } + fn record_tick_receipt_batch( + &mut self, + correlations: &[(ReceiptCorrelationRecord, WalTickDecision)], + action_outcomes_by_submission: &BTreeMap, + state_delta: &WalRuntimeStateDeltaRecord, + state_delta_digest: Hash, + ) -> Result { + let mut correlations = correlations.to_vec(); + correlations.sort_by_key(|(correlation, _)| correlation.ingress_id); + let Some((first_correlation, _)) = correlations.first() else { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + }; + if correlations + .windows(2) + .any(|pair| pair[0].0.ingress_id == pair[1].0.ingress_id) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let same_tick = correlations.iter().all(|(correlation, _)| { + correlation.head_key == first_correlation.head_key + && correlation.commit_global_tick == first_correlation.commit_global_tick + && correlation.worldline_tick_after == first_correlation.worldline_tick_after + && correlation.tick_receipt_digest == first_correlation.tick_receipt_digest + && correlation.commit_hash == first_correlation.commit_hash + }); + if !same_tick { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let first_has_action_outcome = + action_outcomes_by_submission.contains_key(&first_correlation.submission_id); + if correlations.iter().any(|(correlation, _)| { + action_outcomes_by_submission.contains_key(&correlation.submission_id) + != first_has_action_outcome + }) { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + + let mut next_receipt_frontier = self.receipt_frontier_digest; + let mut records = Vec::with_capacity(correlations.len()); + for (correlation, decision) in &correlations { + let receipt = TickReceiptRecord { + receipt_ref: correlation.causal_receipt_ref, + decision: *decision, + }; + let wal_correlation = WalReceiptCorrelationRecord { + receipt_ref: correlation.causal_receipt_ref, + causal_parent_receipts: correlation.causal_parent_receipts.clone(), + }; + next_receipt_frontier = + receipt_frontier_digest(next_receipt_frontier, receipt, &wal_correlation); + let action_outcome = action_outcomes_by_submission + .get(&correlation.submission_id) + .map(|outcome| { + if wal_tick_decision_for_action_outcome(outcome) != *decision { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + retain_action_outcome_v1( + correlation.submission_id, + correlation.ingress_id, + outcome, + ) + .map_err(TrustedRuntimeWalError::from) + }) + .transpose()?; + records.push((receipt, wal_correlation, action_outcome)); + } + let next_runtime_frontier = runtime_state_frontier_digest( + self.runtime_state_frontier_digest, + first_correlation, + state_delta_digest, + ); + let transaction = build_replayable_tick_batch_transaction( + self.builder( + WalTransactionKind::SchedulerTick, + WalAppendAuthority::TrustedScheduler, + WalTransactionId::from_hash(tick_batch_transaction_digest( + &correlations, + state_delta_digest, + )), + ), + records, + state_delta.to_payload_bytes()?, + vec![ + AffectedFrontier { + kind: AffectedFrontierKind::ReceiptIndex, + before_digest: self.receipt_frontier_digest, + after_digest: next_receipt_frontier, + }, + AffectedFrontier { + kind: AffectedFrontierKind::RuntimeState, + before_digest: self.runtime_state_frontier_digest, + after_digest: next_runtime_frontier, + }, + ], + )?; + let commit = self.append_transaction(transaction)?; + self.receipt_frontier_digest = next_receipt_frontier; + self.runtime_state_frontier_digest = next_runtime_frontier; + Ok(commit) + } + fn record_executable_operation_installation( &mut self, retained_installation_bytes: &[u8], @@ -2860,13 +3569,20 @@ impl TrustedRuntimeWalCursor { submission_frontier_digest(cursor.submission_frontier_digest, record); } WalTransactionKind::SchedulerTick => { - let (receipt, correlation, state_delta_digest, provenance_entry) = - tick_records_from_transaction(transaction)?; - cursor.receipt_frontier_digest = receipt_frontier_digest( - cursor.receipt_frontier_digest, - receipt, - &correlation, - ); + let (records, state_delta_digest, provenance_entry) = + tick_record_batch_from_transaction(transaction)?; + let Some((_, first_correlation, _)) = records.first() else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + for (receipt, correlation, _) in &records { + cursor.receipt_frontier_digest = receipt_frontier_digest( + cursor.receipt_frontier_digest, + *receipt, + correlation, + ); + } cursor.runtime_state_frontier_digest = match provenance_entry { Some(entry) => runtime_state_frontier_digest_from_fields( cursor.runtime_state_frontier_digest, @@ -2880,7 +3596,7 @@ impl TrustedRuntimeWalCursor { ), None => recovered_legacy_runtime_state_frontier_digest( cursor.runtime_state_frontier_digest, - correlation, + first_correlation.clone(), state_delta_digest, ), }; @@ -3390,11 +4106,36 @@ fn operation_application_basis_matches_scope_v1( } } +fn recovered_worldline_state_at<'a>( + cache: &'a mut BTreeMap<(crate::WorldlineId, crate::WorldlineTick), crate::WorldlineState>, + recovered_provenance: &ProvenanceService, + worldline_id: crate::WorldlineId, + frontier_state: &crate::WorldlineState, + worldline_tick: crate::WorldlineTick, + failure_detail: &'static str, +) -> Result<&'a crate::WorldlineState, TrustedRuntimeWalError> { + let coordinate = (worldline_id, worldline_tick); + if let std::collections::btree_map::Entry::Vacant(entry) = cache.entry(coordinate) { + let state = recovered_provenance + .replay_worldline_state_at(worldline_id, frontier_state, worldline_tick) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: failure_detail, + })?; + entry.insert(state); + } + cache + .get(&coordinate) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: failure_detail, + }) +} + fn validate_recovered_echo_operation_parent_states( runtime: &WorldlineRuntime, recovered_provenance: &ProvenanceService, recovery: &TrustedRuntimeWalRecovery, -) -> Result<(), TrustedRuntimeWalError> { +) -> Result { + let mut recovered_states = BTreeMap::new(); let installations = recovery .installed_echo_operations .iter() @@ -3414,11 +4155,14 @@ fn validate_recovered_echo_operation_parent_states( detail: "operation receipt names an unavailable recovery worldline", }, )?; - let parent_state = recovered_provenance - .replay_worldline_state_at(worldline_id, frontier.state(), basis.worldline_tick()) - .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "operation receipt parent state cannot be reconstructed", - })?; + let parent_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + worldline_id, + frontier.state(), + basis.worldline_tick(), + "operation receipt parent state cannot be reconstructed", + )?; let entry = entries.get(&(worldline_id, basis.worldline_tick())).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "operation receipt has no recovered state transition", @@ -3439,7 +4183,7 @@ fn validate_recovered_echo_operation_parent_states( patch, receipt.installed_operation_id().as_hash(), installed.program(), - &parent_state, + parent_state, ); let application_basis_matches = exact_scope.is_some_and(|node| { operation_application_basis_matches_scope_v1( @@ -3456,7 +4200,463 @@ fn validate_recovered_echo_operation_parent_states( }); } } - Ok(()) + + let submissions = recovery + .witnessed_submissions + .records() + .iter() + .map(|record| (record.submission.submission_id, record)) + .collect::>(); + let correlations = recovery + .receipt_correlations + .iter() + .map(|correlation| (correlation.submission_id, correlation)) + .collect::>(); + let outcomes = recovery + .echo_operation_action_outcomes + .iter() + .map(|(submission_id, _, outcome)| (*submission_id, outcome)) + .collect::>(); + let mut reconstructed_preparations = BTreeMap::new(); + let mut reconstructed_evaluations = BTreeMap::new(); + let mut action_scopes = BTreeMap::new(); + let mut action_tick_members = BTreeMap::< + ( + crate::WriterHeadKey, + crate::WorldlineTick, + crate::GlobalTick, + Hash, + ), + Vec<(Hash, Hash)>, + >::new(); + for (submission_id, ingress_id, outcome) in &recovery.echo_operation_action_outcomes { + let submission = submissions.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no retained submission", + }, + )?; + let correlation = correlations.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no retained receipt correlation", + }, + )?; + let invocation_bytes = echo_operation_action_invocation_bytes_v1(&submission.envelope) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no canonical invocation", + })?; + let invocation = inspect_action_invocation_v1(invocation_bytes).map_err(|_| { + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome invocation cannot be inspected", + } + })?; + action_scopes.insert(*submission_id, invocation.scope); + let installed = installations.get(&invocation.package_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no recovered installation", + }, + )?; + let basis = invocation.evaluation_basis; + let basis_worldline_id = basis.writer_head().worldline_id; + let frontier = runtime.worldlines().get(&basis_worldline_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis names an unavailable recovery worldline", + }, + )?; + let basis_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + basis_worldline_id, + frontier.state(), + basis.worldline_tick(), + "Action basis state cannot be reconstructed", + )?; + if basis_state.state_root() != basis.state_root() + || !action_application_basis_matches_state_v1(installed, invocation_bytes, basis_state) + .unwrap_or(false) + || !evaluation_basis_matches_recovered_coordinate(basis, &entries) + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis disagrees with reconstructed causal state", + }); + } + let tick_before = correlation + .worldline_tick_after + .as_u64() + .checked_sub(1) + .map(crate::WorldlineTick::from_raw) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has an impossible Tick coordinate", + })?; + let basis_is_current = + basis.writer_head() == correlation.head_key && basis.worldline_tick() == tick_before; + let basis_changed = matches!( + outcome, + EchoOperationActionOutcomeV1::Obstructed(obstruction) + if obstruction.kind() == crate::EchoOperationObstructionKindV1::BasisChanged + ); + if basis_changed == basis_is_current { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis posture disagrees with the scheduler Tick parent", + }); + } + let transition = entries + .get(&(correlation.head_key.worldline_id, tick_before)) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no recovered scheduler transition", + })?; + let policy_id = transition + .patch + .as_ref() + .map(crate::WorldlineTickPatchV1::policy_id) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome scheduler transition has no retained patch", + })?; + let reconstructed = match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + let prepared = reconstruct_action_preparation_v1( + installed, + invocation_bytes, + receipt.retained_invocation_admission_maximum_budget(), + receipt.retained_invocation_admission_policy_id(), + receipt.invocation_admission_id(), + basis_state, + policy_id, + ) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "committed Action preparation cannot be reconstructed", + })?; + if prepared.package_id() != receipt.package_id() + || prepared.installed_operation_id() != receipt.installed_operation_id() + || prepared.invocation_id() != receipt.invocation_id() + || prepared.evaluation_basis().identity() != receipt.evaluation_basis_id() + || prepared.private_evaluation_id() != receipt.private_evaluation_id() + || prepared.prepared_patch_digest() != receipt.prepared_patch_digest() + || prepared.result_id() != receipt.prepared_result_id() + || prepared.actual_footprint_digest() != receipt.actual_footprint_digest() + || prepared.preparation_id() != receipt.preparation_id() + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: + "committed Action evidence disagrees with reconstructed preparation", + }); + } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Prepared(prepared.clone()), + ); + Some(prepared) + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + let prepared = reconstruct_action_preparation_v1( + installed, + invocation_bytes, + conflict.invocation_admission_maximum_budget, + conflict.invocation_admission_policy_id, + conflict.invocation_admission_id, + basis_state, + policy_id, + ) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflicting Action preparation cannot be reconstructed", + })?; + if prepared.package_id() != conflict.package_id + || prepared.installed_operation_id() != conflict.installed_operation_id + || prepared.invocation_id() != conflict.invocation_id + || prepared.evaluation_basis().identity() != conflict.evaluation_basis_id + || prepared.private_evaluation_id() != conflict.private_evaluation_id + || prepared.prepared_patch_digest() != conflict.prepared_patch_digest + || prepared.result_id() != conflict.prepared_result_id + || prepared.actual_footprint_digest() != conflict.actual_footprint_digest + || prepared.preparation_id() != conflict.preparation_id + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflict evidence disagrees with reconstructed preparation", + }); + } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Prepared(prepared.clone()), + ); + Some(prepared) + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + if basis_changed { + if !action_admission_evidence_matches_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + ) { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "basis obstruction has invalid invocation-admission evidence", + }); + } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Obstructed(obstruction.clone()), + ); + None + } else { + let evaluation = reconstruct_action_evaluation_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + basis_state, + policy_id, + ) + .ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "obstructed Action evaluation cannot be reconstructed", + }, + )?; + match &evaluation { + EchoOperationPreparationV1::Obstructed(reconstructed) + if reconstructed.has_same_predecision_evidence(obstruction) => {} + EchoOperationPreparationV1::Prepared(_) + if obstruction.kind() + == crate::EchoOperationObstructionKindV1::BudgetExceeded => {} + _ => { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: + "obstruction evidence disagrees with reconstructed evaluation", + }) + } + } + let prepared = match &evaluation { + EchoOperationPreparationV1::Prepared(prepared) => Some(prepared.clone()), + EchoOperationPreparationV1::Obstructed(_) => None, + }; + reconstructed_evaluations.insert(*submission_id, evaluation); + prepared + } + } + }; + if let Some(prepared) = reconstructed { + reconstructed_preparations.insert(*submission_id, prepared); + } + action_tick_members + .entry(( + correlation.head_key, + correlation.worldline_tick_after, + correlation.commit_global_tick, + correlation.commit_hash, + )) + .or_default() + .push((*ingress_id, *submission_id)); + } + for ((head_key, worldline_tick_after, commit_global_tick, _), mut members) in + action_tick_members + { + members.sort_by_key(|(ingress_id, _)| *ingress_id); + let tick_before = worldline_tick_after + .as_u64() + .checked_sub(1) + .map(crate::WorldlineTick::from_raw) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has an impossible parent coordinate", + })?; + let transition = entries.get(&(head_key.worldline_id, tick_before)).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has no recovered state transition", + }, + )?; + let policy_id = transition + .patch + .as_ref() + .map(crate::WorldlineTickPatchV1::policy_id) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has no retained replay patch", + })?; + let frontier = runtime.worldlines().get(&head_key.worldline_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick names an unavailable recovery worldline", + }, + )?; + let mut reconstructed_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + head_key.worldline_id, + frontier.state(), + tick_before, + "Action Tick parent state cannot be reconstructed", + )? + .clone(); + let candidates = members + .iter() + .map( + |(ingress_id, submission_id)| -> Result<_, TrustedRuntimeWalError> { + let outcome = outcomes.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no recovered outcome", + }, + )?; + let rule_id = match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + receipt.installed_operation_id().as_hash() + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + obstruction.installed_operation_id().as_hash() + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + conflict.installed_operation_id.as_hash() + } + }; + Ok(SchedulerEchoOperationCandidateV1 { + submission_id: *submission_id, + ingress_id: *ingress_id, + scope: *action_scopes.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no reconstructed scope", + }, + )?, + rule_id, + preparation: reconstructed_evaluations + .get(submission_id) + .cloned() + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no reconstructed evaluation", + })?, + }) + }, + ) + .collect::, _>>()?; + let reconstructed_batch = commit_scheduler_action_batch_to_state_v1( + candidates, + head_key, + &mut reconstructed_state, + commit_global_tick, + policy_id, + ) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick composition cannot be reconstructed", + })?; + let reconstructed_outcomes = reconstructed_batch + .outcomes + .into_iter() + .collect::>(); + if members.iter().any(|(_, submission_id)| { + reconstructed_outcomes.get(submission_id) != outcomes.get(submission_id).copied() + }) { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome disagrees with reconstructed Tick composition", + }); + } + let ordered_rule_ids = members + .iter() + .map(|(_, submission_id)| match outcomes.get(submission_id) { + Some(EchoOperationActionOutcomeV1::Committed(receipt)) => { + Ok(receipt.installed_operation_id().as_hash()) + } + Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) => { + Ok(obstruction.installed_operation_id().as_hash()) + } + Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) => { + Ok(conflict.installed_operation_id.as_hash()) + } + None => Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick member has no recovered outcome", + }), + }) + .collect::, _>>()?; + let applied_preparations = members + .iter() + .filter_map(|(_, submission_id)| { + if matches!( + outcomes.get(submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + ) { + reconstructed_preparations + .get(submission_id) + .map(Box::as_ref) + } else { + None + } + }) + .collect::>(); + let reconstructed_patch = action_batch_patch_from_preparations_v1( + policy_id, + &ordered_rule_ids, + &applied_preparations, + ); + if reconstructed_patch.digest() != transition.expected.patch_digest { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick patch omits or alters a reconstructed applied member", + }); + } + for (index, (_, submission_id)) in members.iter().enumerate() { + let Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) = + outcomes.get(submission_id) + else { + continue; + }; + let candidate = reconstructed_preparations.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflicting Action has no reconstructed preparation", + }, + )?; + let mut expected_blockers = Vec::new(); + for (earlier_index, (_, earlier_submission_id)) in members[..index].iter().enumerate() { + if !matches!( + outcomes.get(earlier_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + ) { + continue; + } + let earlier = reconstructed_preparations + .get(earlier_submission_id) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "committed Action has no reconstructed preparation", + })?; + if crate::engine_impl::footprints_conflict( + candidate.actual_footprint(), + earlier.actual_footprint(), + ) { + expected_blockers.push(u32::try_from(earlier_index).map_err(|_| { + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action Tick has too many members to index", + } + })?); + } + } + if expected_blockers != conflict.blocked_by { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflicting Action blockers disagree with reconstructed footprints", + }); + } + } + } + Ok(recovered_states.len()) +} + +fn evaluation_basis_matches_recovered_coordinate( + basis: EchoOperationEvaluationBasisV1, + entries: &BTreeMap<(crate::WorldlineId, crate::WorldlineTick), &ProvenanceEntry>, +) -> bool { + if basis.worldline_tick() == crate::WorldlineTick::ZERO { + return basis.commit_global_tick().is_none() + && basis.commit_id() + == crate::echo_operation::genesis_commit_id( + basis.writer_head(), + basis.state_root(), + ); + } + let Some(parent_tick) = basis.worldline_tick().as_u64().checked_sub(1) else { + return false; + }; + entries + .get(&( + basis.writer_head().worldline_id, + crate::WorldlineTick::from_raw(parent_tick), + )) + .is_some_and(|parent| { + parent.head_key == Some(basis.writer_head()) + && parent.expected.state_root == basis.state_root() + && parent.expected.commit_hash == basis.commit_id() + && basis.commit_global_tick() == Some(parent.commit_global_tick) + }) } fn validate_operation_receipt_state_delta( @@ -3576,6 +4776,9 @@ fn validate_operation_receipt_parent_material( struct RecoveredRuntimeStateMaterial { provenance_entries: Vec, receipt_correlations: Vec, + echo_operation_action_outcomes: Vec<(Hash, Hash, EchoOperationActionOutcomeV1)>, + echo_operation_action_decisions: BTreeMap, + echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder, missing_runtime_state_deltas: Vec, } @@ -3584,17 +4787,28 @@ fn recover_runtime_state_delta_material( ) -> Result { let mut entries_by_coordinate = BTreeMap::new(); let mut correlations_by_submission = BTreeMap::new(); + let mut action_outcomes_by_submission = BTreeMap::new(); + let mut action_decisions_by_submission = BTreeMap::new(); + let mut action_installation_count_before_tick = BTreeMap::new(); + let mut installation_ordinals = BTreeMap::new(); let mut submission_by_ticket = BTreeMap::new(); let mut missing = Vec::new(); for transaction in &report.transactions { + if transaction.commit.transaction_kind + == WalTransactionKind::ExecutableOperationInstallation + { + let (installed, _) = operation_installation_from_transaction(transaction)?; + let next_ordinal = installation_ordinals.len(); + installation_ordinals + .entry(installed.package_id()) + .or_insert(next_ordinal); + continue; + } if transaction.commit.transaction_kind == WalTransactionKind::ExecutableOperationTick { let (_, state_delta, _) = operation_tick_records_from_transaction(transaction)?; let entry = state_delta.provenance_entry().clone(); let coordinate = (entry.worldline_id, entry.worldline_tick); - if entries_by_coordinate - .get(&coordinate) - .is_some_and(|existing| existing != &entry) - { + if entries_by_coordinate.contains_key(&coordinate) { return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { worldline_id: entry.worldline_id, worldline_tick: entry.worldline_tick, @@ -3606,89 +4820,105 @@ fn recover_runtime_state_delta_material( if transaction.commit.transaction_kind != WalTransactionKind::SchedulerTick { continue; } - let receipt = tick_receipt_from_transaction(transaction)?; - let wal_correlation = tick_correlation_from_transaction(transaction)?; - if wal_correlation.receipt_ref != receipt.receipt_ref { - return Err(decode_trusted_runtime_wal_payload( - WalDecodeError::InvalidEmbeddedFrame, - )); - } + let (records, _, _) = tick_record_batch_from_transaction(transaction)?; let state_delta_frame = required_unique_transaction_frame( transaction, WalRecordKind::RuntimeStateDeltaRecorded, )?; if state_delta_frame.payload.canonical_bytes.len() == core::mem::size_of::() { - missing.push(receipt.receipt_ref.identity_digest()); + missing.extend( + records + .iter() + .map(|(receipt, _, _)| receipt.receipt_ref.identity_digest()), + ); continue; } let state_delta = WalRuntimeStateDeltaRecord::from_payload_bytes( &state_delta_frame.payload.canonical_bytes, )?; - if state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest { + if records.iter().any(|(receipt, _, _)| { + state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest + }) { return Err(RetainedProvenanceError::Inconsistent("state-delta receipt").into()); } let entry = state_delta.provenance_entry().clone(); let head_key = entry .head_key .ok_or(RetainedProvenanceError::MissingHeadKey)?; - let expected_receipt_ref = crate::CausalTickReceiptRef { - worldline_id: entry.worldline_id, - worldline_tick_after: entry - .worldline_tick - .checked_add(1) - .ok_or(RetainedProvenanceError::Inconsistent("worldline tick"))?, - commit_global_tick: entry.commit_global_tick, - commit_hash: entry.expected.commit_hash, - submission_id: receipt.receipt_ref.submission_id, - ticket_digest: receipt.receipt_ref.ticket_digest, - receipt_content_digest: state_delta.receipt_digest(), - }; - if receipt.receipt_ref != expected_receipt_ref { - return Err(RetainedProvenanceError::Inconsistent("causal receipt ref").into()); - } - let persistence = ReceiptCorrelationPersistenceRecord { - submission_id: receipt.receipt_ref.submission_id, - ticket_digest: receipt.receipt_ref.ticket_digest, - causal_receipt_ref: receipt.receipt_ref, - head_key, - commit_global_tick: entry.commit_global_tick, - worldline_tick_after: receipt.receipt_ref.worldline_tick_after, - tick_receipt_digest: receipt.receipt_ref.receipt_content_digest, - commit_hash: entry.expected.commit_hash, - contract: state_delta.contract().cloned(), - causal_parent_receipts: wal_correlation.causal_parent_receipts, - }; - if correlations_by_submission - .get(&receipt.receipt_ref.submission_id) - .is_some_and(|existing| existing != &persistence) - { - return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { - worldline_id: entry.worldline_id, - worldline_tick: entry.worldline_tick, - }); - } - if submission_by_ticket - .insert( - receipt.receipt_ref.ticket_digest, - receipt.receipt_ref.submission_id, - ) - .is_some_and(|existing| existing != receipt.receipt_ref.submission_id) - { + let coordinate = (entry.worldline_id, entry.worldline_tick); + if entries_by_coordinate.contains_key(&coordinate) { return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { worldline_id: entry.worldline_id, worldline_tick: entry.worldline_tick, }); } - correlations_by_submission.insert(receipt.receipt_ref.submission_id, persistence); - let coordinate = (entry.worldline_id, entry.worldline_tick); - if entries_by_coordinate - .get(&coordinate) - .is_some_and(|existing| existing != &entry) - { - return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { + for (receipt, wal_correlation, action_outcome) in records { + let expected_receipt_ref = crate::CausalTickReceiptRef { worldline_id: entry.worldline_id, - worldline_tick: entry.worldline_tick, - }); + worldline_tick_after: entry + .worldline_tick + .checked_add(1) + .ok_or(RetainedProvenanceError::Inconsistent("worldline tick"))?, + commit_global_tick: entry.commit_global_tick, + commit_hash: entry.expected.commit_hash, + submission_id: receipt.receipt_ref.submission_id, + ticket_digest: receipt.receipt_ref.ticket_digest, + receipt_content_digest: state_delta.receipt_digest(), + }; + if receipt.receipt_ref != expected_receipt_ref { + return Err(RetainedProvenanceError::Inconsistent("causal receipt ref").into()); + } + let persistence = ReceiptCorrelationPersistenceRecord { + submission_id: receipt.receipt_ref.submission_id, + ticket_digest: receipt.receipt_ref.ticket_digest, + causal_receipt_ref: receipt.receipt_ref, + head_key, + commit_global_tick: entry.commit_global_tick, + worldline_tick_after: receipt.receipt_ref.worldline_tick_after, + tick_receipt_digest: receipt.receipt_ref.receipt_content_digest, + commit_hash: entry.expected.commit_hash, + contract: state_delta.contract().cloned(), + causal_parent_receipts: wal_correlation.causal_parent_receipts, + }; + if correlations_by_submission + .get(&receipt.receipt_ref.submission_id) + .is_some_and(|existing| existing != &persistence) + { + return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { + worldline_id: entry.worldline_id, + worldline_tick: entry.worldline_tick, + }); + } + if submission_by_ticket + .insert( + receipt.receipt_ref.ticket_digest, + receipt.receipt_ref.submission_id, + ) + .is_some_and(|existing| existing != receipt.receipt_ref.submission_id) + { + return Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { + worldline_id: entry.worldline_id, + worldline_tick: entry.worldline_tick, + }); + } + correlations_by_submission.insert(receipt.receipt_ref.submission_id, persistence); + if let Some((submission_id, ingress_id, outcome)) = action_outcome { + let decision_conflicts = action_decisions_by_submission + .get(&submission_id) + .is_some_and(|existing| *existing != receipt.decision); + if submission_id != receipt.receipt_ref.submission_id + || action_outcomes_by_submission + .get(&submission_id) + .is_some_and(|existing| existing != &(ingress_id, outcome.clone())) + || decision_conflicts + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + action_decisions_by_submission.insert(submission_id, receipt.decision); + action_outcomes_by_submission.insert(submission_id, (ingress_id, outcome)); + action_installation_count_before_tick + .insert(submission_id, installation_ordinals.len()); + } } entries_by_coordinate.insert(coordinate, entry); } @@ -3711,9 +4941,19 @@ fn recover_runtime_state_delta_material( }); missing.sort_unstable(); missing.dedup(); + let echo_operation_action_outcomes = action_outcomes_by_submission + .into_iter() + .map(|(submission_id, (ingress_id, outcome))| (submission_id, ingress_id, outcome)) + .collect(); Ok(RecoveredRuntimeStateMaterial { provenance_entries: entries, receipt_correlations: correlations, + echo_operation_action_outcomes, + echo_operation_action_decisions: action_decisions_by_submission, + echo_operation_action_installation_order: RecoveredEchoOperationActionInstallationOrder { + package_ordinals: installation_ordinals, + installation_count_before_tick: action_installation_count_before_tick, + }, missing_runtime_state_deltas: missing, }) } @@ -3743,6 +4983,332 @@ fn validate_recovered_causal_parent_evidence( Ok(()) } +fn validate_recovered_echo_operation_action_outcomes( + witnessed_submissions: &WitnessedSubmissionPersistenceSnapshot, + receipt_correlations: &[ReceiptCorrelationPersistenceRecord], + provenance_entries: &[ProvenanceEntry], + installed_echo_operations: &[InstalledEchoOperationV1], + outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], + decisions: &BTreeMap, + installation_order: &RecoveredEchoOperationActionInstallationOrder, +) -> Result { + #[allow(unused_mut)] + let mut metrics = RecoveredEchoOperationActionOutcomeValidationMetrics::default(); + let envelopes = witnessed_submissions + .records() + .iter() + .map(|record| (record.submission.submission_id, &record.envelope)) + .collect::>(); + let submissions = witnessed_submissions + .records() + .iter() + .map(|record| (record.submission.submission_id, &record.submission)) + .collect::>(); + let correlations = receipt_correlations + .iter() + .map(|correlation| (correlation.submission_id, correlation)) + .collect::>(); + let provenance = provenance_entries + .iter() + .filter_map(|entry| { + Some(( + ( + entry.head_key?, + entry.worldline_tick.checked_add(1)?, + entry.commit_global_tick, + entry.expected.commit_hash, + ), + entry, + )) + }) + .collect::>(); + let installed_by_package = installed_echo_operations + .iter() + .map(|installed| (installed.package_id(), installed)) + .collect::>(); + let mut outcomes_by_tick = BTreeMap::< + ( + crate::WriterHeadKey, + crate::WorldlineTick, + crate::GlobalTick, + Hash, + ), + Vec<( + Hash, + &EchoOperationActionOutcomeV1, + &ReceiptCorrelationPersistenceRecord, + crate::echo_operation::EchoOperationActionInvocationEvidenceV1, + )>, + >::new(); + for (submission_id, ingress_id, outcome) in outcomes { + if decisions.get(submission_id) != Some(&wal_tick_decision_for_action_outcome(outcome)) { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let envelope = envelopes + .get(submission_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let submission = submissions + .get(submission_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let correlation = correlations + .get(submission_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let invocation_bytes = echo_operation_action_invocation_bytes_v1(envelope) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let invocation = inspect_action_invocation_v1(invocation_bytes) + .map_err(|_| TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + #[cfg(any(test, feature = "host_test"))] + { + metrics.indexed_package_lookups += 1; + } + let installed = installed_by_package + .get(&invocation.package_id) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + #[cfg(any(test, feature = "host_test"))] + { + metrics.installation_order_lookups += 2; + } + let installed_before_tick = installation_order + .installation_count_before_tick + .get(submission_id) + .zip( + installation_order + .package_ordinals + .get(&invocation.package_id), + ) + .is_some_and(|(count, ordinal)| ordinal < count); + if !installed_before_tick { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + if envelope.ingress_id() != *ingress_id { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let invocation_admission_id = match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + Some(receipt.invocation_admission_id()) + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + Some(obstruction.invocation_admission_id()) + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + Some(conflict.invocation_admission_id) + } + }; + if correlation.ticket_digest != correlation.causal_receipt_ref.ticket_digest + || invocation_admission_id.is_some_and(|admission_id| { + correlation.ticket_digest + != echo_operation_action_admission_digest_from_parts( + submission, + invocation.package_id, + installed.installed_operation_id(), + admission_id, + ) + }) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + if validate_receipt_installation_v1(receipt, installed).is_err() + || receipt.tick_receipt_digest() != correlation.tick_receipt_digest + || receipt.commit_id() != correlation.commit_hash + || receipt.commit_global_tick() != Some(correlation.commit_global_tick) + || receipt.worldline_tick_after() != correlation.worldline_tick_after + || receipt.evaluation_basis().writer_head() != correlation.head_key + || receipt.package_id() != invocation.package_id + || receipt.invocation_id() != invocation.invocation_id + || receipt.invocation_bytes_digest() != invocation.invocation_bytes_digest + || receipt.evaluation_basis() != invocation.evaluation_basis + || receipt.terminal_posture() != EchoOperationTerminalPostureV1::Committed + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + if obstruction.installed_operation_id() != installed.installed_operation_id() + || obstruction.package_id() != invocation.package_id + || !action_admission_evidence_matches_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + ) + || obstruction.invocation_id() != invocation.invocation_id + || obstruction.evaluation_basis_id() != invocation.evaluation_basis.identity() + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + if conflict.package_id != invocation.package_id + || conflict.installed_operation_id != installed.installed_operation_id() + || !action_admission_evidence_matches_v1( + installed, + invocation_bytes, + conflict.invocation_admission_maximum_budget, + conflict.invocation_admission_policy_id, + conflict.invocation_admission_id, + ) + || conflict.evaluation_basis_id != invocation.evaluation_basis.identity() + || !action_preparation_identity_matches_v1( + conflict.private_evaluation_id, + conflict.prepared_patch_digest, + conflict.prepared_result_id, + conflict.preparation_id, + ) + || conflict.blocked_by.is_empty() + || conflict + .blocked_by + .windows(2) + .any(|pair| pair[0] >= pair[1]) + || conflict.invocation_id != invocation.invocation_id + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + } + outcomes_by_tick + .entry(( + correlation.head_key, + correlation.worldline_tick_after, + correlation.commit_global_tick, + correlation.commit_hash, + )) + .or_default() + .push((*ingress_id, outcome, correlation, invocation)); + } + for (tick_coordinate, mut group) in outcomes_by_tick { + let entry = provenance + .get(&tick_coordinate) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let tick_receipt = entry + .tick_receipt + .as_ref() + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + group.sort_by_key(|(ingress_id, _, _, _)| *ingress_id); + if group.len() != tick_receipt.entries().len() { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let committed_receipts = group + .iter() + .filter_map(|(_, outcome, _, _)| match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => Some(receipt.as_ref()), + EchoOperationActionOutcomeV1::Obstructed(_) + | EchoOperationActionOutcomeV1::RejectedFootprintConflict(_) => None, + }) + .collect::>(); + if !committed_receipts.is_empty() { + let expected_composition_digest = action_batch_composition_digest_from_receipts_v1( + &committed_receipts, + entry.expected.patch_digest, + ); + if committed_receipts + .iter() + .any(|receipt| receipt.composition_digest() != Some(expected_composition_digest)) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + for (index, (_, outcome, correlation, invocation)) in group.into_iter().enumerate() { + if entry.commit_global_tick != correlation.commit_global_tick + || entry.head_key != Some(correlation.head_key) + || entry.worldline_id != correlation.head_key.worldline_id + || tick_receipt.digest() != correlation.tick_receipt_digest + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + let tick_entry = tick_receipt + .entries() + .get(index) + .ok_or(TrustedRuntimeWalError::SchedulerTickBatchMismatch)?; + let blockers = tick_receipt.blocked_by(index); + if tick_entry.scope != invocation.scope + || tick_entry.scope_hash + != crate::scope_hash(&tick_entry.rule_id, &tick_entry.scope) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + if tick_entry.disposition != TickReceiptDisposition::Applied + || !blockers.is_empty() + || tick_entry.rule_id != receipt.installed_operation_id().as_hash() + || receipt.state_root_after() != entry.expected.state_root + || receipt.committed_patch_digest() != Some(entry.expected.patch_digest) + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + let decision_coordinate_matches = + obstruction.decision_coordinate().is_some_and(|coordinate| { + action_decision_coordinate_matches_correlation_v1( + coordinate, + correlation, + index, + ) + }); + if tick_entry.disposition + != TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) + || !blockers.is_empty() + || tick_entry.rule_id != obstruction.installed_operation_id().as_hash() + || !decision_coordinate_matches + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + let decision_coordinate_matches = + conflict.decision_coordinate().is_some_and(|coordinate| { + action_decision_coordinate_matches_correlation_v1( + coordinate, + correlation, + index, + ) + }); + if tick_entry.disposition + != TickReceiptDisposition::Rejected(TickReceiptRejection::FootprintConflict) + || tick_entry.rule_id != conflict.installed_operation_id.as_hash() + || blockers != conflict.blocked_by + || !decision_coordinate_matches + { + return Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch); + } + } + } + } + } + Ok(metrics) +} + +fn action_decision_coordinate_matches_correlation_v1( + coordinate: crate::EchoOperationActionDecisionCoordinateV1, + correlation: &ReceiptCorrelationPersistenceRecord, + member_index: usize, +) -> bool { + u32::try_from(member_index).is_ok_and(|member_index| { + coordinate.writer_head() == correlation.head_key + && coordinate.worldline_tick_after() == correlation.worldline_tick_after + && coordinate.commit_global_tick() == correlation.commit_global_tick + && coordinate.commit_id() == correlation.commit_hash + && coordinate.tick_receipt_digest() == correlation.tick_receipt_digest + && coordinate.member_index() == member_index + }) +} + +fn wal_tick_decision_for_action_outcome(outcome: &EchoOperationActionOutcomeV1) -> WalTickDecision { + match outcome { + EchoOperationActionOutcomeV1::Committed(_) => WalTickDecision::Applied, + EchoOperationActionOutcomeV1::Obstructed(_) => WalTickDecision::Obstructed, + EchoOperationActionOutcomeV1::RejectedFootprintConflict(_) => { + WalTickDecision::RejectedFootprintConflict + } + } +} + fn operation_installation_from_transaction( transaction: &crate::causal_wal::WalRecoveredTransaction, ) -> Result<(InstalledEchoOperationV1, Vec), TrustedRuntimeWalError> { @@ -3789,6 +5355,7 @@ fn operation_tick_records_from_transaction( Ok((receipt, state_delta, state_delta_digest)) } +#[cfg(test)] fn tick_records_from_transaction( transaction: &crate::causal_wal::WalRecoveredTransaction, ) -> Result< @@ -3800,15 +5367,163 @@ fn tick_records_from_transaction( ), TrustedRuntimeWalError, > { - let receipt = tick_receipt_from_transaction(transaction)?; - let correlation = tick_correlation_from_transaction(transaction)?; - if correlation.receipt_ref != receipt.receipt_ref { + let (mut records, state_delta_digest, provenance_entry) = + tick_record_batch_from_transaction(transaction)?; + if records.len() != 1 { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let (receipt, correlation, action_outcome) = records.remove(0); + if action_outcome.is_some() { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + Ok((receipt, correlation, state_delta_digest, provenance_entry)) +} + +type RecoveredSchedulerTickMember = ( + TickReceiptRecord, + WalReceiptCorrelationRecord, + Option<(Hash, Hash, EchoOperationActionOutcomeV1)>, +); +type RecoveredSchedulerTickBatch = ( + Vec, + Hash, + Option, +); + +fn tick_record_batch_from_transaction( + transaction: &crate::causal_wal::WalRecoveredTransaction, +) -> Result { + let state_delta_index = transaction + .frames + .iter() + .position(|frame| frame.header.record_kind == WalRecordKind::RuntimeStateDeltaRecorded) + .ok_or_else(missing_trusted_runtime_record)?; + if state_delta_index == 0 || state_delta_index + 1 != transaction.frames.len() { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let mut records = Vec::new(); + let first_frame = &transaction.frames[0]; + if first_frame.header.record_kind != WalRecordKind::TickReceiptRecorded { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + if tick_receipt_payload_is_batch(&first_frame.payload.canonical_bytes) { + let receipts = decode_tick_receipt_records(&first_frame.payload.canonical_bytes) + .map_err(decode_trusted_runtime_wal_payload)?; + let mut index = 1; + for receipt in receipts { + let Some(correlation_frame) = transaction.frames.get(index) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + let Some(outcome_frame) = transaction.frames.get(index + 1) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + if correlation_frame.header.record_kind != WalRecordKind::ReceiptCorrelationRecorded + || outcome_frame.header.record_kind + != WalRecordKind::ExecutableOperationActionOutcomeRecorded + { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let correlation = WalReceiptCorrelationRecord::from_payload_bytes( + &correlation_frame.payload.canonical_bytes, + ) + .map_err(decode_trusted_runtime_wal_payload)?; + let action_outcome = recover_action_outcome_v1(&outcome_frame.payload.canonical_bytes)?; + if correlation.receipt_ref != receipt.receipt_ref + || action_outcome.0 != receipt.receipt_ref.submission_id + { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + records.push((receipt, correlation, Some(action_outcome))); + index += 2; + } + if index != state_delta_index { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + } else { + let mut index = 0; + while index < state_delta_index { + let Some(receipt_frame) = transaction.frames.get(index) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + let Some(correlation_frame) = transaction.frames.get(index + 1) else { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + }; + if receipt_frame.header.record_kind != WalRecordKind::TickReceiptRecorded + || correlation_frame.header.record_kind != WalRecordKind::ReceiptCorrelationRecorded + { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + let receipt = + TickReceiptRecord::from_payload_bytes(&receipt_frame.payload.canonical_bytes) + .map_err(decode_trusted_runtime_wal_payload)?; + let correlation = WalReceiptCorrelationRecord::from_payload_bytes( + &correlation_frame.payload.canonical_bytes, + ) + .map_err(decode_trusted_runtime_wal_payload)?; + if correlation.receipt_ref != receipt.receipt_ref { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + index += 2; + if transaction.frames.get(index).is_some_and(|frame| { + frame.header.record_kind == WalRecordKind::ExecutableOperationActionOutcomeRecorded + }) { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + records.push((receipt, correlation, None)); + } + } + let action_outcome_count = records + .iter() + .filter(|(_, _, action_outcome)| action_outcome.is_some()) + .count(); + if action_outcome_count != 0 && action_outcome_count != records.len() { return Err(decode_trusted_runtime_wal_payload( WalDecodeError::InvalidEmbeddedFrame, )); } - let state_delta_frame = - required_unique_transaction_frame(transaction, WalRecordKind::RuntimeStateDeltaRecorded)?; + if action_outcome_count != 0 { + let mut previous_ingress_id = None; + for (_, _, action_outcome) in &records { + let (_, ingress_id, _) = action_outcome.as_ref().ok_or_else(|| { + decode_trusted_runtime_wal_payload(WalDecodeError::InvalidEmbeddedFrame) + })?; + if previous_ingress_id.is_some_and(|previous| previous >= *ingress_id) { + return Err(decode_trusted_runtime_wal_payload( + WalDecodeError::InvalidEmbeddedFrame, + )); + } + previous_ingress_id = Some(*ingress_id); + } + } + let state_delta_frame = &transaction.frames[state_delta_index]; let (state_delta_digest, provenance_entry) = if state_delta_frame.payload.canonical_bytes.len() == core::mem::size_of::() { @@ -3825,7 +5540,9 @@ fn tick_records_from_transaction( let state_delta = WalRuntimeStateDeltaRecord::from_payload_bytes( &state_delta_frame.payload.canonical_bytes, )?; - if state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest { + if records.iter().any(|(receipt, _, _)| { + state_delta.receipt_digest() != receipt.receipt_ref.receipt_content_digest + }) { return Err(RetainedProvenanceError::Inconsistent("state-delta receipt").into()); } ( @@ -3833,25 +5550,7 @@ fn tick_records_from_transaction( Some(state_delta.provenance_entry().clone()), ) }; - Ok((receipt, correlation, state_delta_digest, provenance_entry)) -} - -fn tick_receipt_from_transaction( - transaction: &crate::causal_wal::WalRecoveredTransaction, -) -> Result { - let receipt_frame = - required_unique_transaction_frame(transaction, WalRecordKind::TickReceiptRecorded)?; - TickReceiptRecord::from_payload_bytes(&receipt_frame.payload.canonical_bytes) - .map_err(decode_trusted_runtime_wal_payload) -} - -fn tick_correlation_from_transaction( - transaction: &crate::causal_wal::WalRecoveredTransaction, -) -> Result { - let correlation_frame = - required_unique_transaction_frame(transaction, WalRecordKind::ReceiptCorrelationRecorded)?; - WalReceiptCorrelationRecord::from_payload_bytes(&correlation_frame.payload.canonical_bytes) - .map_err(decode_trusted_runtime_wal_payload) + Ok((records, state_delta_digest, provenance_entry)) } fn required_unique_transaction_frame( @@ -4104,7 +5803,19 @@ impl TrustedRuntimeApp<'_> { &mut self, envelope: IngressEnvelope, ) -> Result { - self.host.runtime.submit_app_intent(envelope) + let is_echo_operation_action = + echo_operation_action_invocation_bytes_v1(&envelope).is_some(); + if is_echo_operation_action && self.host.runtime_wal.is_some() { + return Err(RuntimeError::EchoOperationActionRequiresRuntimeWalAck); + } + let handle = self.host.runtime.submit_app_intent(envelope)?; + if self.host.runtime_wal.is_none() { + self.host.track_pending_echo_operation_action_v1( + handle.submission_id, + is_echo_operation_action, + ); + } + Ok(handle) } /// Submits canonical intent material and returns only after the configured @@ -4136,6 +5847,8 @@ impl TrustedRuntimeApp<'_> { return Err(TrustedRuntimeHostError::RuntimeWalUnavailable); } + let is_echo_operation_action = + echo_operation_action_invocation_bytes_v1(&envelope).is_some(); let before_runtime = self.host.runtime.clone(); let handle = match admission { AppIntentAdmission::Ordinary => { @@ -4150,27 +5863,31 @@ impl TrustedRuntimeApp<'_> { self.host.runtime = before_runtime; return Err(TrustedRuntimeHostError::RuntimeWalUnavailable); }; - if handle.duplicate { - match runtime_wal.has_submission_acceptance(handle.submission_id, envelope.ingress_id()) - { - Ok(true) => return Ok(handle), - Ok(false) => {} - Err(error) => { - self.host.runtime = before_runtime; - return Err(error.into()); - } - } + if handle.duplicate + && runtime_wal.has_submission_acceptance(handle.submission_id, envelope.ingress_id()) + { + self.host.track_pending_echo_operation_action_v1( + handle.submission_id, + is_echo_operation_action, + ); + return Ok(handle); } if let Err(error) = runtime_wal.record_submission_acceptance(&envelope, handle) { if runtime_wal.recover_filesystem_submission_acceptance_after_error( handle.submission_id, envelope.ingress_id(), ) { + self.host.track_pending_echo_operation_action_v1( + handle.submission_id, + is_echo_operation_action, + ); return Ok(handle); } self.host.runtime = before_runtime; return Err(error.into()); } + self.host + .track_pending_echo_operation_action_v1(handle.submission_id, is_echo_operation_action); Ok(handle) } @@ -4374,6 +6091,36 @@ fn provider_contract_host_admission_digest( hasher.finalize().into() } +fn echo_operation_action_admission_digest( + submission: &IntentSubmissionRecord, + admitted: &AdmittedEchoOperationInvocationV1, +) -> Hash { + echo_operation_action_admission_digest_from_parts( + submission, + admitted.package_id(), + admitted.installed_operation_id(), + admitted.admission_id(), + ) +} + +fn echo_operation_action_admission_digest_from_parts( + submission: &IntentSubmissionRecord, + package_id: crate::EchoOperationPackageIdV1, + installed_operation_id: crate::InstalledEchoOperationIdV1, + invocation_admission_id: crate::EchoOperationInvocationAdmissionIdV1, +) -> Hash { + let mut hasher = blake3::Hasher::new(); + hasher.update(ECHO_OPERATION_ACTION_ADMISSION_DIGEST_DOMAIN); + hasher.update(&submission.submission_id); + hasher.update(&submission.ingress_id); + hasher.update(submission.head_key.worldline_id.as_bytes()); + hasher.update(submission.head_key.head_id.as_bytes()); + hasher.update(&package_id.as_hash()); + hasher.update(&installed_operation_id.as_hash()); + hasher.update(&invocation_admission_id.as_hash()); + hasher.finalize().into() +} + fn submission_frontier_digest(previous: Hash, record: SubmissionAcceptanceRecord) -> Hash { let mut hasher = blake3::Hasher::new(); hasher.update(TRUSTED_RUNTIME_WAL_DOMAIN); @@ -4414,6 +6161,10 @@ fn wal_tick_decision_from_observation( reason: TickReceiptRejection::FootprintConflict, .. } => WalTickDecision::RejectedFootprintConflict, + IntentOutcomeDecision::Rejected { + reason: TickReceiptRejection::ExecutableOperationObstruction, + .. + } => WalTickDecision::Obstructed, IntentOutcomeDecision::NoMatchingReceiptEntry { .. } => { return Err(TrustedRuntimeWalError::TickOutcomeUnavailable { submission_id: correlation.submission_id, @@ -4432,6 +6183,24 @@ mod tests { }; use bytes::Bytes; + type BorrowedActionOutcomeBatchWriter = + for<'a> fn( + &mut TrustedRuntimeWal, + &[(ReceiptCorrelationRecord, WalTickDecision)], + &BTreeMap, + &WalRuntimeStateDeltaRecord, + Hash, + ) -> Result; + + #[test] + fn tick_receipt_batch_accepts_borrowed_action_outcomes() { + let writer: BorrowedActionOutcomeBatchWriter = TrustedRuntimeWal::record_tick_receipt_batch; + assert!(std::ptr::fn_addr_eq( + writer, + TrustedRuntimeWal::record_tick_receipt_batch as BorrowedActionOutcomeBatchWriter + )); + } + fn test_head_key() -> WriterHeadKey { WriterHeadKey { worldline_id: WorldlineId::from_bytes([9; 32]), @@ -4572,6 +6341,60 @@ mod tests { .expect("the recovery parent-state fixture is lawful") } + #[test] + fn executable_operation_index_preserves_legacy_root_without_action_outcomes() { + let operation_coordinate = "echo.test.LegacyRecoveryIndex.v1"; + let authority_profile_identity = [0x17; 32]; + let budget = crate::EchoOperationBudgetV1::new(7, 1_024, 1_024); + let package = crate::ExecutableOperationPackageV1::new( + operation_coordinate, + crate::EchoOperationSemanticClosureV1::new( + [0x10; 32], + [0x11; 32], + [0x12; 32], + [0x13; 32], + "echo.test.legacy-index-schema/v1", + [0x14; 32], + "echo.test.legacy-index-lawpack/v1", + [0x15; 32], + ), + crate::echo_operation_target_profile_identity_v1(), + authority_profile_identity, + budget, + crate::EchoOperationProgramV1::anchored_node_attachment_compare_and_set( + crate::make_type_id("legacy-index-node"), + crate::make_type_id("legacy-index-attachment"), + 128, + ), + ); + let package_bytes = package + .to_canonical_bytes() + .expect("the legacy-index package is canonical"); + let package_id = crate::echo_operation_package_id_v1(&package_bytes); + let admitted = admit_package_v1( + &crate::EchoOperationAdmissionPolicyV1::exact( + package_id, + operation_coordinate, + authority_profile_identity, + budget, + ), + package_bytes, + ) + .expect("the legacy-index package is admitted"); + let installed = + installed_from_admitted(admitted).expect("the legacy-index package installs"); + + assert_eq!( + recovered_echo_operation_index_root([0x20; 32], &[installed], &[], &[]) + .expect("the legacy index root is computable"), + [ + 0xff, 0x77, 0x8e, 0x79, 0x1c, 0x4b, 0x7f, 0x99, 0xb7, 0xa5, 0x4c, 0x4b, 0x8d, 0xdb, + 0x36, 0x91, 0x62, 0x17, 0x8a, 0x22, 0xe2, 0xbe, 0xde, 0xc6, 0x54, 0x4c, 0x8d, 0x25, + 0xa1, 0xa0, 0x69, 0x04, + ] + ); + } + #[test] fn creation_wal_scope_accepts_descendants_and_rejects_mutated_shapes() { let node = crate::NodeKey { @@ -5013,16 +6836,28 @@ mod tests { let provenance = BTreeMap::from([(parent_coordinate, &parent)]); validate_operation_receipt_parent_material(basis, &child, &provenance) .expect("the exact retained parent corroborates every causal basis field"); + assert!(evaluation_basis_matches_recovered_coordinate( + basis, + &provenance + )); let mut wrong_root = parent.clone(); wrong_root.expected.state_root = [39; 32]; let provenance = BTreeMap::from([(parent_coordinate, &wrong_root)]); assert!(validate_operation_receipt_parent_material(basis, &child, &provenance).is_err()); + assert!(!evaluation_basis_matches_recovered_coordinate( + basis, + &provenance + )); let mut wrong_global_tick = parent.clone(); wrong_global_tick.commit_global_tick = GlobalTick::from_raw(9); let provenance = BTreeMap::from([(parent_coordinate, &wrong_global_tick)]); assert!(validate_operation_receipt_parent_material(basis, &child, &provenance).is_err()); + assert!(!evaluation_basis_matches_recovered_coordinate( + basis, + &provenance + )); } fn test_correlation(receipt_digest: Hash) -> ReceiptCorrelationRecord { @@ -5587,6 +7422,69 @@ mod tests { } } + #[test] + fn runtime_wal_legacy_tick_rejects_action_outcome_frame() { + let wal = TrustedRuntimeWal::new_in_memory().expect("test WAL should initialize"); + let receipt = TickReceiptRecord { + receipt_ref: CausalTickReceiptRef { + worldline_id: WorldlineId::from_bytes([40; 32]), + worldline_tick_after: WorldlineTick::from_raw(1), + commit_global_tick: GlobalTick::from_raw(1), + commit_hash: [41; 32], + submission_id: [42; 32], + ticket_digest: [43; 32], + receipt_content_digest: [44; 32], + }, + decision: WalTickDecision::Obstructed, + }; + let correlation = WalReceiptCorrelationRecord { + receipt_ref: receipt.receipt_ref, + causal_parent_receipts: Vec::new(), + }; + let action_outcome = vec![0xa5]; + + let mut builder = wal.builder( + WalTransactionKind::SchedulerTick, + WalAppendAuthority::TrustedScheduler, + WalTransactionId::from_hash([51; 32]), + ); + builder + .push_record( + WalRecordKind::TickReceiptRecorded, + receipt.to_payload_bytes(), + ) + .expect("fixture legacy receipt must append"); + builder + .push_record( + WalRecordKind::ReceiptCorrelationRecorded, + correlation.to_payload_bytes(), + ) + .expect("fixture legacy correlation must append"); + builder + .push_record( + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + action_outcome, + ) + .expect("adversarial Action outcome must append structurally"); + builder + .push_record(WalRecordKind::RuntimeStateDeltaRecorded, [52; 32].to_vec()) + .expect("fixture state delta must append"); + let transaction = builder + .commit(Vec::new()) + .expect("adversarial legacy transaction must commit structurally"); + let recovered = crate::causal_wal::WalRecoveredTransaction { + commit: transaction.commit, + frames: transaction.frames, + }; + + assert!(matches!( + tick_record_batch_from_transaction(&recovered), + Err(TrustedRuntimeWalError::Recovery(WalRecoveryError::Index( + WalRecoveryIndexError::Decode(WalDecodeError::InvalidEmbeddedFrame) + ))) + )); + } + #[test] fn recovered_correlation_rejects_parents_not_bound_by_envelope() { let head_key = test_head_key(); @@ -5653,6 +7551,25 @@ fn tick_transaction_digest( hasher.finalize().into() } +fn tick_batch_transaction_digest( + correlations: &[(ReceiptCorrelationRecord, WalTickDecision)], + state_delta_digest: Hash, +) -> Hash { + let mut hasher = blake3::Hasher::new(); + hasher.update(TRUSTED_RUNTIME_WAL_DOMAIN); + hasher.update(b"tick-batch-transaction:v1\0"); + hasher.update(&(correlations.len() as u64).to_le_bytes()); + for (correlation, decision) in correlations { + hasher.update(&correlation.ticketed_ingress_id); + hasher.update(&correlation.causal_receipt_ref.to_canonical_bytes()); + hasher.update(&correlation.ingress_id); + hash_causal_parent_receipts(&mut hasher, &correlation.causal_parent_receipts); + hasher.update(&[wal_tick_decision_code(*decision)]); + } + hasher.update(&state_delta_digest); + hasher.finalize().into() +} + fn receipt_frontier_digest( previous: Hash, receipt: TickReceiptRecord, @@ -5800,6 +7717,7 @@ struct RecoveredRuntimeWalIndexEvidence<'a> { causal_anchor_history: &'a [WitnessedCausalAnchorAdmission], installed_echo_operations: &'a [InstalledEchoOperationV1], echo_operation_receipts: &'a [EchoOperationReceiptV1], + echo_operation_action_outcomes: &'a [(Hash, Hash, EchoOperationActionOutcomeV1)], } fn runtime_wal_recovery_certificate( @@ -5839,6 +7757,7 @@ fn recovered_runtime_wal_indexes_root( causal_anchor_root, indexes.installed_echo_operations, indexes.echo_operation_receipts, + indexes.echo_operation_action_outcomes, ) } @@ -5846,6 +7765,29 @@ fn recovered_echo_operation_index_root( base_root: Hash, installations: &[InstalledEchoOperationV1], receipts: &[EchoOperationReceiptV1], + action_outcomes: &[(Hash, Hash, EchoOperationActionOutcomeV1)], +) -> Result { + let legacy_root = + recovered_echo_operation_legacy_index_root(base_root, installations, receipts)?; + if action_outcomes.is_empty() { + return Ok(legacy_root); + } + let mut hasher = blake3::Hasher::new(); + hasher.update(b"echo:trusted-runtime-wal:executable-operation-index:v2\0"); + hasher.update(&legacy_root); + hasher.update(&(action_outcomes.len() as u64).to_le_bytes()); + for (submission_id, ingress_id, outcome) in action_outcomes { + let bytes = retain_action_outcome_v1(*submission_id, *ingress_id, outcome)?; + hasher.update(&(bytes.len() as u64).to_le_bytes()); + hasher.update(&bytes); + } + Ok(hasher.finalize().into()) +} + +fn recovered_echo_operation_legacy_index_root( + base_root: Hash, + installations: &[InstalledEchoOperationV1], + receipts: &[EchoOperationReceiptV1], ) -> Result { if installations.is_empty() && receipts.is_empty() { return Ok(base_root); diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 42d8de85..7975e3f0 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -16,18 +16,23 @@ use warp_core::causal_wal::{ WalAppendAuthority, WalBuildError, WalDurabilityMode, WalRecordKind, WalSegmentId, WalTransactionBuilder, WalTransactionId, WalTransactionKind, WalValidationError, WriterEpochId, }; +#[cfg(feature = "host_test")] +use warp_core::causal_wal::{FilesystemWalFaultPlan, FilesystemWalFaultTarget, WalTickDecision}; +#[cfg(feature = "host_test")] +use warp_core::EchoOperationCommitErrorV1; use warp_core::{ make_head_id, make_node_id, make_type_id, make_warp_id, AtomPayload, AttachmentValue, - EchoOperationAdmissionErrorKindV1, EchoOperationAdmissionPolicyV1, - EchoOperationAnchoredNodeOccupancyV1, EchoOperationApplicationBasisV1, - EchoOperationArtifactErrorKindV1, EchoOperationBudgetV1, + EchoOperationActionOutcomeV1, EchoOperationAdmissionErrorKindV1, + EchoOperationAdmissionPolicyV1, EchoOperationAnchoredNodeOccupancyV1, + EchoOperationApplicationBasisV1, EchoOperationArtifactErrorKindV1, EchoOperationBudgetV1, EchoOperationInvocationAdmissionErrorKindV1, EchoOperationInvocationAdmissionPolicyV1, EchoOperationInvocationV1, EchoOperationObstructionKindV1, EchoOperationPreparationV1, EchoOperationProgramV1, EchoOperationSemanticClosureV1, EchoOperationTerminalPostureV1, - EngineBuilder, ExecutableOperationPackageV1, GraphStore, InboxPolicy, InstalledEchoOperationV1, - NodeKey, NodeRecord, PlaybackMode, RuntimeWalActivationGap, SchedulerKind, TrustedRuntimeHost, - TrustedRuntimeHostError, TrustedRuntimeWalConfig, TrustedRuntimeWalError, WorldlineId, - WorldlineRuntime, WorldlineState, WriterHead, WriterHeadKey, + EngineBuilder, ExecutableOperationPackageV1, GraphStore, HeadEligibility, InboxPolicy, + IngressTarget, InstalledEchoOperationV1, NodeKey, NodeRecord, PlaybackMode, RuntimeError, + RuntimeWalActivationGap, SchedulerKind, TrustedRuntimeHost, TrustedRuntimeHostError, + TrustedRuntimeWalConfig, TrustedRuntimeWalError, WorldlineId, WorldlineRuntime, WorldlineState, + WriterHead, WriterHeadKey, ACTION_BATCH_CANDIDATE_LIMIT_V1, }; const OPERATION_COORDINATE: &str = "echo.fixture.SetAnchoredAtom.v1"; @@ -102,24 +107,44 @@ fn operation_frontier(kind: AffectedFrontierKind, label: &str) -> AffectedFronti } fn fixture_host() -> (TrustedRuntimeHost, WriterHeadKey, NodeKey) { + let (host, head_key, nodes) = fixture_host_with_independent_nodes(1); + ( + host, + head_key, + *nodes.first().expect("the fixture contains its root node"), + ) +} + +fn fixture_host_with_independent_nodes( + node_count: usize, +) -> (TrustedRuntimeHost, WriterHeadKey, Vec) { + assert!(node_count > 0, "the fixture must retain one root node"); let warp_id = make_warp_id("operation-fixture"); - let node_id = make_node_id("operation-fixture-root"); + let root_id = make_node_id("operation-fixture-root"); let node_type = make_type_id("operation-fixture-node"); let attachment_type = make_type_id("operation-fixture-atom"); - let node = NodeKey { - warp_id, - local_id: node_id, - }; + let nodes = (0..node_count) + .map(|index| NodeKey { + warp_id, + local_id: if index == 0 { + root_id + } else { + make_node_id(&format!("operation-fixture-node-{index}")) + }, + }) + .collect::>(); let mut store = GraphStore::new(warp_id); - store.insert_node(node_id, NodeRecord { ty: node_type }); - store.set_node_attachment( - node_id, - Some(AttachmentValue::Atom(AtomPayload::new( - attachment_type, - Bytes::from_static(b"before"), - ))), - ); - let state = WorldlineState::from_root_store(store, node_id) + for node in &nodes { + store.insert_node(node.local_id, NodeRecord { ty: node_type }); + store.set_node_attachment( + node.local_id, + Some(AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"before"), + ))), + ); + } + let state = WorldlineState::from_root_store(store, root_id) .expect("the fixture state has one lawful root"); let worldline_id = WorldlineId::from_bytes(digest("operation-fixture-worldline")); let head_key = WriterHeadKey { @@ -154,7 +179,72 @@ fn fixture_host() -> (TrustedRuntimeHost, WriterHeadKey, NodeKey) { .build(); let host = TrustedRuntimeHost::new(runtime, engine) .expect("the trusted Echo runtime host initializes"); - (host, head_key, node) + (host, head_key, nodes) +} + +fn fixture_host_with_two_worldlines() -> (TrustedRuntimeHost, WriterHeadKey, WriterHeadKey, NodeKey) +{ + let warp_id = make_warp_id("operation-fixture"); + let node_id = make_node_id("operation-fixture-root"); + let node = NodeKey { + warp_id, + local_id: node_id, + }; + let mut store = GraphStore::new(warp_id); + store.insert_node( + node_id, + NodeRecord { + ty: make_type_id("operation-fixture-node"), + }, + ); + store.set_node_attachment( + node_id, + Some(AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))), + ); + let state = WorldlineState::from_root_store(store, node_id) + .expect("the fixture state has one lawful root"); + let first_head = WriterHeadKey { + worldline_id: WorldlineId::from_bytes(digest("operation-fixture-worldline-first")), + head_id: make_head_id("operation-fixture-writer"), + }; + let second_head = WriterHeadKey { + worldline_id: WorldlineId::from_bytes(digest("operation-fixture-worldline-second")), + head_id: make_head_id("operation-fixture-writer"), + }; + let mut runtime = WorldlineRuntime::new(); + for head in [first_head, second_head] { + runtime + .register_worldline(head.worldline_id, state.clone()) + .expect("the fixture worldline registers"); + runtime + .register_writer_head(WriterHead::with_routing( + head, + PlaybackMode::Play, + InboxPolicy::AcceptAll, + None, + true, + )) + .expect("the fixture writer registers"); + } + + let mut engine_store = GraphStore::default(); + let engine_root = make_node_id("root"); + engine_store.insert_node( + engine_root, + NodeRecord { + ty: make_type_id("world"), + }, + ); + let engine = EngineBuilder::new(engine_store, engine_root) + .scheduler(SchedulerKind::Radix) + .workers(1) + .build(); + let host = TrustedRuntimeHost::new(runtime, engine) + .expect("the trusted Echo runtime host initializes"); + (host, first_head, second_head, node) } /// Like [`fixture_host`], but the warp-scoped store also contains a second, @@ -167,6 +257,20 @@ fn fixture_host_with_bare_node( second_node_local_id: warp_core::NodeId, second_node_type: warp_core::TypeId, second_attachment: Option<(warp_core::TypeId, &'static [u8])>, +) -> (TrustedRuntimeHost, WriterHeadKey, NodeKey, NodeKey) { + fixture_host_with_bare_node_and_policy( + second_node_local_id, + second_node_type, + second_attachment, + InboxPolicy::AcceptAll, + ) +} + +fn fixture_host_with_bare_node_and_policy( + second_node_local_id: warp_core::NodeId, + second_node_type: warp_core::TypeId, + second_attachment: Option<(warp_core::TypeId, &'static [u8])>, + inbox_policy: InboxPolicy, ) -> (TrustedRuntimeHost, WriterHeadKey, NodeKey, NodeKey) { let warp_id = make_warp_id("operation-fixture"); let node_id = make_node_id("operation-fixture-root"); @@ -219,7 +323,7 @@ fn fixture_host_with_bare_node( .register_writer_head(WriterHead::with_routing( head_key, PlaybackMode::Play, - InboxPolicy::AcceptAll, + inbox_policy, None, true, )) @@ -472,6 +576,33 @@ fn canonical_invocation( .expect("fixture invocation encodes canonically") } +fn action_invocation( + host: &TrustedRuntimeHost, + installed: &InstalledEchoOperationV1, + head_key: WriterHeadKey, + node: NodeKey, + current: &[u8], + replacement: &[u8], +) -> Vec { + let attachment_type = make_type_id("operation-fixture-atom"); + let application_basis = warp_core::echo_operation_anchored_node_application_basis_v1( + node, + attachment_type, + current, + ); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis) + .expect("Echo resolves the shared parent and candidate application basis"); + canonical_invocation( + installed, + evaluation_basis, + node, + warp_core::echo_operation_atom_value_digest_v1(attachment_type, current), + replacement.to_vec(), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + ) +} + fn replace_canonical_map_field( bytes: &[u8], field_name: &str, @@ -1523,6 +1654,10 @@ fn operation_wal_codes_append_without_renumbering_legacy_evidence() { WalRecordKind::ExecutableOperationStateDeltaRecorded.stable_code(), 27 ); + assert_eq!( + WalRecordKind::ExecutableOperationActionOutcomeRecorded.stable_code(), + 28 + ); assert_eq!(AffectedFrontierKind::RuntimeState.stable_code(), 2); assert_eq!(AffectedFrontierKind::CausalAnchorIndex.stable_code(), 8); @@ -2541,3 +2676,1663 @@ fn update_precondition_refuses_when_the_node_has_the_wrong_type() { EchoOperationObstructionKindV1::NodeTypeMismatch ); } + +#[test] +fn budget_deferred_executable_action_reaches_typed_basis_obstruction() { + let second_node_id = make_node_id("operation-fixture-budget-deferred"); + let attachment_type = make_type_id("operation-fixture-atom"); + let (mut host, head_key, first_node, second_node) = fixture_host_with_bare_node_and_policy( + second_node_id, + make_type_id("operation-fixture-node"), + Some((attachment_type, b"second-before")), + InboxPolicy::Budgeted { max_per_tick: 1 }, + ); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let first = action_invocation( + &host, + &installed, + head_key, + first_node, + b"before", + b"first-after", + ); + let second = action_invocation( + &host, + &installed, + head_key, + second_node, + b"second-before", + b"second-after", + ); + let first_submission = host + .app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + first, + ) + .expect("the first invocation is one canonical Action"), + ) + .expect("the first Action is accepted"); + let second_submission = host + .app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + second, + ) + .expect("the second invocation is one canonical Action"), + ) + .expect("the second Action is accepted"); + let first_submission_id = first_submission.submission_id; + let second_submission_id = second_submission.submission_id; + + let first_tick = host + .tick_once() + .expect("one Action is selected at the original basis"); + assert_eq!(first_tick.len(), 1); + assert_eq!(first_tick[0].admitted_count, 1); + + let (committed_submission_id, deferred_submission_id) = match ( + host.echo_operation_action_outcome_v1(&first_submission_id), + host.echo_operation_action_outcome_v1(&second_submission_id), + ) { + (Some(EchoOperationActionOutcomeV1::Committed(_)), None) => { + (first_submission_id, second_submission_id) + } + (None, Some(EchoOperationActionOutcomeV1::Committed(_))) => { + (second_submission_id, first_submission_id) + } + outcomes => panic!("exactly one Action must commit in the first Tick: {outcomes:?}"), + }; + + let second_tick = host + .tick_once() + .expect("the deferred Action reaches scheduler-owned private evaluation"); + assert_eq!(second_tick.len(), 1); + assert_eq!(second_tick[0].admitted_count, 1); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&deferred_submission_id) + else { + panic!("the stale deferred Action must receive one typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); + assert!(matches!( + host.echo_operation_action_outcome_v1(&committed_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + +#[test] +fn scheduler_commits_two_independent_executable_actions_in_one_durable_tick() { + let wal_dir = TempWalDir::new(); + let second_node_id = make_node_id("operation-fixture-second"); + let attachment_type = make_type_id("operation-fixture-atom"); + let package_id; + let first_submission_id; + let second_submission_id; + + { + let (mut host, head_key, first_node, second_node) = fixture_host_with_bare_node( + second_node_id, + make_type_id("operation-fixture-node"), + Some((attachment_type, b"second-before")), + ); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the scheduler fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + package_id = installed.package_id(); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let first_invocation = action_invocation( + &host, + &installed, + head_key, + first_node, + b"before", + b"first-after", + ); + let second_invocation = action_invocation( + &host, + &installed, + head_key, + second_node, + b"second-before", + b"second-after", + ); + let first_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + first_invocation, + ) + .expect("the first invocation is one canonical Action envelope"); + let second_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + second_invocation, + ) + .expect("the second invocation is one canonical Action envelope"); + + first_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(first_envelope) + .expect("the first Action is durable before acknowledgement") + .submission_id; + second_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(second_envelope) + .expect("the second Action is durable before acknowledgement") + .submission_id; + assert_eq!(host.runtime().pending_witnessed_submission_count(), 2); + + assert_eq!( + host.runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline remains live") + .state() + .current_tick() + .as_u64(), + 0, + "submission and runtime admission cannot evaluate the Action" + ); + + let steps = host + .tick_once() + .expect("the scheduler privately evaluates and commits both Actions"); + let host = TrustedRuntimeHost::from_parts(host.into_parts()); + assert_eq!(steps.len(), 1); + assert_eq!(steps[0].admitted_count, 2); + assert_eq!(steps[0].worldline_tick_after.as_u64(), 1); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 0); + + for submission_id in [first_submission_id, second_submission_id] { + let Some(EchoOperationActionOutcomeV1::Committed(receipt)) = + host.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("each Action has one typed committed outcome"); + }; + assert_eq!(receipt.worldline_tick_after().as_u64(), 1); + } + + let state = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the committed worldline remains available") + .state(); + assert_eq!( + state + .store(&first_node.warp_id) + .and_then(|store| store.node_attachment(&first_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"first-after"), + ))) + ); + assert_eq!( + state + .store(&second_node.warp_id) + .and_then(|store| store.node_attachment(&second_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"second-after"), + ))) + ); + let runtime_wal = host + .runtime_wal() + .expect("the scheduler WAL remains enabled"); + let commits = runtime_wal.commits(); + assert_eq!( + commits + .iter() + .map(|commit| commit.transaction_kind) + .collect::>(), + vec![ + WalTransactionKind::ExecutableOperationInstallation, + WalTransactionKind::SubmissionIntake, + WalTransactionKind::SubmissionIntake, + WalTransactionKind::SchedulerTick, + ], + "one scheduler-owned Tick must be one WAL transaction" + ); + let scheduler_transaction_id = commits + .last() + .expect("the scheduler Tick has one commit marker") + .transaction_id; + assert_eq!( + runtime_wal + .frames() + .into_iter() + .filter(|frame| frame.header.transaction_id == scheduler_transaction_id) + .map(|frame| frame.header.record_kind) + .collect::>(), + vec![ + WalRecordKind::TickReceiptRecorded, + WalRecordKind::ReceiptCorrelationRecorded, + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + WalRecordKind::ReceiptCorrelationRecorded, + WalRecordKind::ExecutableOperationActionOutcomeRecorded, + WalRecordKind::RuntimeStateDeltaRecorded, + ], + "one composite Tick retains one decision record, per-Action outcomes, and one state delta" + ); + + #[cfg(feature = "host_test")] + { + assert!(matches!( + runtime_wal.recover_with_repeated_scheduler_tick_for_test(), + Err(TrustedRuntimeWalError::RuntimeStateDeltaConflict { .. }) + )); + let mut adversarial = runtime_wal + .recover_read_only() + .expect("the honest composite Tick recovers"); + let (indexed_package_lookups, installation_order_lookups) = adversarial + .echo_operation_action_installation_lookup_counts_for_test() + .expect("the honest composite Tick outcomes validate"); + assert_eq!( + indexed_package_lookups, 2, + "recovery must perform one indexed package lookup per Action" + ); + assert_eq!( + installation_order_lookups, 4, + "recovery must perform two indexed installation-order lookups per Action" + ); + assert_eq!( + adversarial + .echo_operation_parent_state_replay_count_for_test( + host.runtime(), + host.provenance() + ) + .expect("the honest composite Tick parent states validate"), + 1, + "one shared basis must be reconstructed once for the whole Action batch" + ); + let mut missing_installation = adversarial.clone(); + missing_installation.installed_echo_operations.clear(); + assert!(matches!( + missing_installation.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut contradictory_decision = adversarial.clone(); + contradictory_decision.replace_echo_operation_action_decision_for_test( + first_submission_id, + WalTickDecision::Obstructed, + ); + assert!(matches!( + contradictory_decision.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut forged_admission_ticket = adversarial.clone(); + let forged_ticket = digest("forged-Action-admission-ticket"); + forged_admission_ticket.receipt_correlations[0].ticket_digest = forged_ticket; + forged_admission_ticket.receipt_correlations[0] + .causal_receipt_ref + .ticket_digest = forged_ticket; + assert!(matches!( + forged_admission_ticket.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut retroactive_installation = adversarial.clone(); + retroactive_installation + .clear_echo_operation_action_installations_before_tick_for_test( + first_submission_id, + ); + assert!(matches!( + retroactive_installation.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut forged_composition = adversarial.clone(); + let Some((_, _, EchoOperationActionOutcomeV1::Committed(receipt))) = forged_composition + .echo_operation_action_outcomes + .iter_mut() + .find(|(_, _, outcome)| { + matches!(outcome, EchoOperationActionOutcomeV1::Committed(_)) + }) + else { + panic!("the recovered composite Tick retains one committed Action receipt"); + }; + receipt.replace_composition_digest_for_test(digest("forged-composition")); + assert!(matches!( + forged_composition.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut omitted_member_patch = runtime_wal + .recover_read_only() + .expect("the honest composite Tick recovers for membership mutation"); + let forged_patch_digest = digest("composite-patch-omits-an-applied-member"); + let transition = omitted_member_patch + .provenance_entries + .iter_mut() + .find(|entry| entry.expected.commit_hash == steps[0].commit_hash) + .expect("the recovered scheduler transition remains available"); + transition.expected.patch_digest = forged_patch_digest; + transition + .patch + .as_mut() + .expect("the scheduler transition retains its replay patch") + .patch_digest = forged_patch_digest; + assert!(matches!( + omitted_member_patch.validate_echo_operation_parent_states_for_test( + host.runtime(), + host.provenance() + ), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); + assert!( + adversarial + .echo_operation_action_outcomes + .windows(2) + .all(|pair| pair[0].1 < pair[1].1), + "Action outcome records are retained in canonical ingress order" + ); + let (first, second) = adversarial.echo_operation_action_outcomes.split_at_mut(1); + std::mem::swap(&mut first[0].2, &mut second[0].2); + assert!(matches!( + adversarial.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + } + } + + let (mut recovered, head_key, first_node, second_node) = fixture_host_with_bare_node( + second_node_id, + make_type_id("operation-fixture-node"), + Some((attachment_type, b"second-before")), + ); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("a fresh host recovers the composite Action Tick"); + assert!(recovered + .engine() + .installed_echo_operation_package_v1(package_id) + .is_some()); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 0); + for submission_id in [first_submission_id, second_submission_id] { + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + } + let state = recovered + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the recovered worldline exists") + .state(); + assert_eq!(state.current_tick().as_u64(), 1); + assert_eq!( + state + .store(&first_node.warp_id) + .and_then(|store| store.node_attachment(&first_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"first-after"), + ))) + ); + assert_eq!( + state + .store(&second_node.warp_id) + .and_then(|store| store.node_attachment(&second_node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + attachment_type, + Bytes::from_static(b"second-after"), + ))) + ); +} + +#[test] +fn scheduler_candidate_limit_leaves_excess_action_pending() { + let candidate_count = ACTION_BATCH_CANDIDATE_LIMIT_V1 + 1; + let (mut host, head_key, nodes) = fixture_host_with_independent_nodes(candidate_count); + host.enable_in_memory_runtime_wal() + .expect("the scheduler budget fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let mut submissions = Vec::new(); + for (index, node) in nodes.into_iter().enumerate() { + let replacement = format!("bounded-after-{index}"); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + replacement.as_bytes(), + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the bounded invocation becomes one canonical Action"); + let ingress_id = envelope.ingress_id(); + let submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the bounded Action is durable") + .submission_id; + submissions.push((ingress_id, submission_id)); + } + assert_eq!( + host.runtime().pending_witnessed_submission_count(), + candidate_count + ); + + let first_tick = host + .tick_once() + .expect("the first scheduler Tick respects its Action candidate limit"); + assert_eq!(first_tick.len(), 1); + assert_eq!( + first_tick[0].admitted_count, + ACTION_BATCH_CANDIDATE_LIMIT_V1 + ); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); + assert_eq!( + submissions + .iter() + .filter(|(_, submission_id)| { + host.echo_operation_action_outcome_v1(submission_id) + .is_some() + }) + .count(), + ACTION_BATCH_CANDIDATE_LIMIT_V1 + ); + submissions.sort_unstable(); + assert!(submissions[..ACTION_BATCH_CANDIDATE_LIMIT_V1] + .iter() + .all(|(_, submission_id)| host + .echo_operation_action_outcome_v1(submission_id) + .is_some())); + assert!(submissions[ACTION_BATCH_CANDIDATE_LIMIT_V1..] + .iter() + .all(|(_, submission_id)| host + .echo_operation_action_outcome_v1(submission_id) + .is_none())); + + let second_tick = host + .tick_once() + .expect("the excess Action remains available to a later scheduler Tick"); + assert_eq!(second_tick.len(), 1); + assert_eq!(second_tick[0].admitted_count, 1); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 0); + assert!(submissions.iter().all(|(_, submission_id)| host + .echo_operation_action_outcome_v1(submission_id) + .is_some())); +} + +#[test] +fn accepted_executable_action_recovers_pending_before_scheduler_evaluation() { + let wal_dir = TempWalDir::new(); + let submission_id; + let package_id; + + { + let (mut host, head_key, node) = fixture_host(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the pending-Action fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + package_id = installed.package_id(); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"after-restart", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes a canonical Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("Action acceptance is durable before acknowledgement") + .submission_id; + assert!(matches!( + host.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); + assert_eq!( + host.runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline remains available") + .state() + .current_tick() + .as_u64(), + 0 + ); + } + + let (mut recovered, head_key, node) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the accepted pre-Tick Action recovers"); + assert!(recovered + .engine() + .installed_echo_operation_package_v1(package_id) + .is_some()); + assert!(matches!( + recovered.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 1); + recovered.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let steps = recovered + .tick_once() + .expect("the scheduler admits and evaluates recovered pending work"); + assert_eq!(steps.len(), 1); + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 0); + let state = recovered + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the recovered Tick advances the worldline") + .state(); + assert_eq!(state.current_tick().as_u64(), 1); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"after-restart"), + ))) + ); +} + +#[test] +fn wal_enabled_host_rejects_action_submission_without_durable_ack() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the non-durable submission fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"must-not-run", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes an Action"); + let error = host + .app() + .submit_intent(envelope) + .expect_err("a WAL-enabled Action must use the durable ACK boundary"); + assert!(matches!( + error, + RuntimeError::EchoOperationActionRequiresRuntimeWalAck + )); + + assert_eq!(host.runtime().pending_witnessed_submission_count(), 0); + let recovery = host + .runtime_wal() + .expect("the runtime WAL remains enabled") + .recover_read_only() + .expect("the WAL contains no dangling scheduler consequence"); + assert!(recovery.witnessed_submissions.is_empty()); + assert!(recovery.echo_operation_action_outcomes.is_empty()); +} + +#[cfg(feature = "host_test")] +#[test] +fn scheduler_action_hot_path_uses_bounded_indexes() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the indexed scheduler fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"indexed-after", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes an Action"); + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the Action acceptance is durable"); + host.runtime_wal() + .expect("the runtime WAL remains enabled") + .reset_recover_read_only_call_count_for_test(); + host.runtime() + .reset_receipt_correlation_full_scan_count_for_test(); + + host.tick_once() + .expect("the indexed Action reaches one scheduler Tick"); + + assert_eq!( + host.runtime_wal() + .expect("the runtime WAL remains enabled") + .recover_read_only_call_count_for_test(), + 0, + "Action admission must not replay all committed WAL history" + ); + assert_eq!( + host.runtime() + .receipt_correlation_full_scan_count_for_test(), + 0, + "Tick persistence must consume newly committed correlations directly" + ); +} + +#[test] +fn typed_action_obstruction_is_durable_and_contributes_no_mutation() { + let wal_dir = TempWalDir::new(); + let submission_id; + let state_root_before; + + { + let (mut host, head_key, node) = fixture_host(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the obstruction fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + state_root_before = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline exists") + .state() + .state_root(); + let application_basis = application_basis(); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis) + .expect("Echo resolves the honest parent basis"); + let invocation = canonical_invocation( + &installed, + evaluation_basis, + node, + digest("intentionally-wrong-value-precondition"), + b"must-not-appear".to_vec(), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the obstructed invocation is still a canonical Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the Action itself is durably accepted") + .submission_id; + let steps = host + .tick_once() + .expect("typed obstruction is a lawful scheduler decision"); + let host = TrustedRuntimeHost::from_parts(host.into_parts()); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("the Action retains its typed evaluator obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::PreconditionMismatch + ); + let decision = obstruction + .decision_coordinate() + .expect("the terminal obstruction names its deciding Tick"); + assert_eq!(decision.writer_head(), head_key); + assert_eq!( + decision.worldline_tick_after(), + steps[0].worldline_tick_after + ); + assert_eq!(decision.commit_id(), steps[0].commit_hash); + assert_eq!(decision.member_index(), 0); + let state = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the decided worldline exists") + .state(); + assert_eq!(state.state_root(), state_root_before); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); + assert!(state + .tick_history() + .last() + .expect("the obstruction is represented by one decided Tick") + .2 + .ops() + .is_empty()); + + #[cfg(feature = "host_test")] + { + let mut adversarial = host + .runtime_wal() + .expect("the obstruction fixture retains its WAL") + .recover_read_only() + .expect("the honest obstruction WAL recovers"); + adversarial.replace_echo_operation_action_obstruction_kind_for_test( + submission_id, + EchoOperationObstructionKindV1::BudgetExceeded, + ); + assert!(matches!( + adversarial.validate_echo_operation_parent_states_for_test( + host.runtime(), + host.provenance() + ), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); + let mut forged_decision = host + .runtime_wal() + .expect("the obstruction fixture retains its WAL") + .recover_read_only() + .expect("the honest obstruction WAL recovers again"); + forged_decision + .replace_echo_operation_action_decision_member_index_for_test(submission_id, 1); + assert!(matches!( + forged_decision.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + } + } + + let (mut recovered, head_key, node) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("a fresh host recovers the typed obstruction"); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + recovered.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("recovery restores the typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::PreconditionMismatch + ); + let recovered_decision = obstruction + .decision_coordinate() + .expect("recovered obstruction retains its deciding Tick"); + assert_eq!(recovered_decision.writer_head(), head_key); + assert_eq!(recovered_decision.member_index(), 0); + let state = recovered + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the recovered worldline exists") + .state(); + assert_eq!(state.state_root(), state_root_before); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); +} + +#[test] +fn unavailable_action_package_does_not_poison_unrelated_scheduler_work() { + let wal_dir = TempWalDir::new(); + let unavailable_submission_id; + let valid_submission_id; + let head_key; + let node; + + { + let (mut host, fixture_head, fixture_node) = fixture_host(); + head_key = fixture_head; + node = fixture_node; + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the admission-obstruction fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let unavailable_package_bytes = operation_package_at( + "echo.fixture.Unavailable.v1", + make_type_id("operation-fixture-node"), + make_type_id("operation-fixture-atom"), + ) + .to_canonical_bytes() + .expect("the unavailable package is canonical"); + let unavailable_package_id = + warp_core::echo_operation_package_id_v1(&unavailable_package_bytes); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis()) + .expect("Echo resolves the current exact parent basis"); + let unavailable_invocation = + EchoOperationInvocationV1::anchored_node_attachment_compare_and_set( + unavailable_package_id, + "echo.fixture.Unavailable.v1", + evaluation_basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + node, + warp_core::echo_operation_atom_value_digest_v1( + make_type_id("operation-fixture-atom"), + b"before", + ), + b"must-not-appear".to_vec(), + ) + .to_canonical_bytes() + .expect("the unavailable-package invocation is canonical"); + unavailable_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + unavailable_invocation, + ) + .expect("the unavailable-package invocation becomes an Action"), + ) + .expect("the unavailable-package Action is durable before acknowledgement") + .submission_id; + + let valid_invocation = + action_invocation(&host, &installed, head_key, node, b"before", b"valid-after"); + valid_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + valid_invocation, + ) + .expect("the valid invocation becomes an Action"), + ) + .expect("the valid Action is durable before acknowledgement") + .submission_id; + + let steps = host + .tick_once() + .expect("one unadmittable Action cannot abort unrelated scheduler work"); + assert_eq!(steps.len(), 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + assert_eq!( + host.echo_operation_action_admission_obstruction_v1(&unavailable_submission_id), + Some(EchoOperationInvocationAdmissionErrorKindV1::OperationUnavailable) + ); + assert!(host + .echo_operation_action_outcome_v1(&unavailable_submission_id) + .is_none()); + assert_eq!(host.runtime().pending_witnessed_submission_count(), 1); + assert!(host + .tick_once() + .expect("quarantined admission work does not poison an idle pass") + .is_empty()); + } + + let (mut recovered, recovered_head, recovered_node) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("fresh-host recovery restores the valid Tick and pending Action"); + recovered.install_echo_operation_action_admission_policy_v1(invocation_policy()); + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + let steps = recovered + .tick_once() + .expect("the recovered unavailable-package Action is quarantined"); + assert!(steps.is_empty()); + assert_eq!( + recovered.echo_operation_action_admission_obstruction_v1(&unavailable_submission_id), + Some(EchoOperationInvocationAdmissionErrorKindV1::OperationUnavailable) + ); + assert!(recovered + .tick_once() + .expect("recovered quarantine remains idle without re-admission") + .is_empty()); + assert_eq!(recovered.runtime().pending_witnessed_submission_count(), 1); + assert_eq!(recovered_head, head_key); + assert_eq!(recovered_node, node); +} + +#[test] +fn run_until_idle_continues_after_admission_only_progress() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the bounded-admission fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let valid_invocation = + action_invocation(&host, &installed, head_key, node, b"before", b"valid-after"); + let valid_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + valid_invocation, + ) + .expect("the valid invocation becomes an Action"); + let valid_ingress_id = valid_envelope.ingress_id(); + + let unavailable_package_bytes = operation_package_at( + "echo.fixture.BoundedUnavailable.v1", + make_type_id("operation-fixture-node"), + make_type_id("operation-fixture-atom"), + ) + .to_canonical_bytes() + .expect("the unavailable package is canonical"); + let unavailable_package_id = + warp_core::echo_operation_package_id_v1(&unavailable_package_bytes); + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis()) + .expect("Echo resolves the current exact parent basis"); + let mut unavailable_envelopes = Vec::new(); + for ordinal in 0..4096_u32 { + let unavailable_invocation = + EchoOperationInvocationV1::anchored_node_attachment_compare_and_set( + unavailable_package_id, + "echo.fixture.BoundedUnavailable.v1", + evaluation_basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + node, + warp_core::echo_operation_atom_value_digest_v1( + make_type_id("operation-fixture-atom"), + b"before", + ), + format!("unavailable-{ordinal}").into_bytes(), + ) + .to_canonical_bytes() + .expect("the unavailable invocation is canonical"); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + unavailable_invocation, + ) + .expect("the unavailable invocation becomes an Action"); + if envelope.ingress_id() < valid_ingress_id { + unavailable_envelopes.push(envelope); + if unavailable_envelopes.len() == ACTION_BATCH_CANDIDATE_LIMIT_V1 { + break; + } + } + } + assert_eq!( + unavailable_envelopes.len(), + ACTION_BATCH_CANDIDATE_LIMIT_V1, + "the deterministic fixture must fill the first bounded admission prefix" + ); + for envelope in unavailable_envelopes { + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("each unavailable Action is durable"); + } + let valid_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(valid_envelope) + .expect("the later valid Action is durable") + .submission_id; + + let report = host + .run_until_idle(4) + .expect("admission-only progress keeps the bounded drain alive"); + assert_eq!(report.committed_steps, 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + +#[test] +fn action_outcome_attribution_ignores_application_controlled_scope_collisions() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the scope-collision fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let valid_invocation = + action_invocation(&host, &installed, head_key, node, b"before", b"after"); + let valid_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + valid_invocation, + ) + .expect("the valid invocation becomes an Action"); + let colliding_scope = NodeKey { + warp_id: node.warp_id, + local_id: warp_core::NodeId(valid_envelope.ingress_id()), + }; + let evaluation_basis = host + .echo_operation_evaluation_basis_v1(head_key, application_basis()) + .expect("Echo resolves the current exact parent basis"); + let obstructed_invocation = + EchoOperationInvocationV1::anchored_node_attachment_compare_and_set( + installed.package_id(), + installed.operation_coordinate(), + evaluation_basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + colliding_scope, + warp_core::echo_operation_atom_value_digest_v1( + make_type_id("operation-fixture-atom"), + b"before", + ), + b"must-not-appear".to_vec(), + ) + .to_canonical_bytes() + .expect("the scope-collision invocation is canonical"); + let obstructed_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + obstructed_invocation, + ) + .expect("the scope-collision invocation becomes an Action"); + + let valid_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(valid_envelope) + .expect("the valid Action is durable") + .submission_id; + let obstructed_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(obstructed_envelope) + .expect("the obstructed Action is durable") + .submission_id; + + let steps = host + .tick_once() + .expect("scope values cannot change positional Action attribution"); + assert_eq!(steps.len(), 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&valid_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&obstructed_submission_id) + else { + panic!("the colliding scope still receives its own typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::NodeMissing + ); +} + +#[test] +fn identical_action_payloads_on_distinct_heads_keep_distinct_outcomes() { + let (mut host, first_head, second_head, node) = fixture_host_with_two_worldlines(); + host.enable_in_memory_runtime_wal() + .expect("the cross-head fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let invocation = action_invocation(&host, &installed, first_head, node, b"before", b"after"); + let first_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: first_head }, + invocation.clone(), + ) + .expect("the first target accepts the canonical Action payload"); + let second_envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: second_head }, + invocation, + ) + .expect("the second target accepts the same canonical Action payload"); + assert_eq!(first_envelope.ingress_id(), second_envelope.ingress_id()); + + let first_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(first_envelope) + .expect("the first target submission is durable") + .submission_id; + let second_submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(second_envelope) + .expect("the second target submission is durable") + .submission_id; + assert_ne!(first_submission_id, second_submission_id); + + let steps = host + .tick_once() + .expect("cross-head duplicate payloads retain distinct Action identities"); + assert_eq!(steps.len(), 2); + assert!(matches!( + host.echo_operation_action_outcome_v1(&first_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&second_submission_id) + else { + panic!("the mismatched target retains its own typed basis obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); +} + +#[test] +fn cross_worldline_basis_obstruction_recovers_on_the_deciding_worldline() { + let wal_dir = TempWalDir::new(); + let submission_id; + + { + let (mut host, basis_head, target_head, node) = fixture_host_with_two_worldlines(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the cross-worldline obstruction WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let invocation = + action_invocation(&host, &installed, basis_head, node, b"before", b"after"); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: target_head }, + invocation, + ) + .expect("the cross-worldline invocation becomes one Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the cross-worldline Action is durable") + .submission_id; + + let steps = host + .tick_once() + .expect("the target worldline decides the basis obstruction"); + assert_eq!(steps.len(), 1); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + host.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("the cross-worldline Action retains one typed obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); + assert_eq!( + obstruction + .decision_coordinate() + .expect("the obstruction names its deciding Tick") + .writer_head(), + target_head + ); + } + + let (mut recovered, _, recovered_target_head, _) = fixture_host_with_two_worldlines(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("fresh-host recovery resolves the target-worldline transition"); + let Some(EchoOperationActionOutcomeV1::Obstructed(obstruction)) = + recovered.echo_operation_action_outcome_v1(&submission_id) + else { + panic!("fresh-host recovery restores the cross-worldline obstruction"); + }; + assert_eq!( + obstruction.kind(), + EchoOperationObstructionKindV1::BasisChanged + ); + assert_eq!( + obstruction + .decision_coordinate() + .expect("the recovered obstruction names its deciding Tick") + .writer_head(), + recovered_target_head + ); +} + +#[test] +fn recovery_separates_identical_commits_on_distinct_worldlines() { + let (mut host, first_head, second_head, node) = fixture_host_with_two_worldlines(); + host.enable_in_memory_runtime_wal() + .expect("the multi-worldline recovery fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let mut submission_ids = Vec::new(); + for head_key in [first_head, second_head] { + let invocation = action_invocation(&host, &installed, head_key, node, b"before", b"after"); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the exact-head invocation becomes an Action"); + submission_ids.push( + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the exact-head Action is durable") + .submission_id, + ); + } + + let steps = host + .tick_once() + .expect("both exact-head Actions commit in one scheduler pass"); + assert_eq!(steps.len(), 2); + assert_eq!(steps[0].commit_hash, steps[1].commit_hash); + + let recovery = host + .runtime_wal() + .expect("the scheduler WAL remains enabled") + .recover_read_only() + .expect("equal commit hashes on distinct worldlines recover independently"); + for submission_id in submission_ids { + assert!(recovery + .echo_operation_action_outcomes + .iter() + .any(|(recovered, _, outcome)| { + *recovered == submission_id + && matches!(outcome, EchoOperationActionOutcomeV1::Committed(_)) + })); + } +} + +#[test] +fn footprint_conflict_recovery_reconstructs_the_rejected_preparation() { + let wal_dir = TempWalDir::new(); + let conflict_submission_id; + + { + let (mut host, head_key, node) = fixture_host(); + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the footprint-conflict fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + let mut submission_ids = Vec::new(); + for replacement in [b"after-a".as_slice(), b"after-b".as_slice()] { + let invocation = + action_invocation(&host, &installed, head_key, node, b"before", replacement); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the conflicting invocation becomes an Action"); + submission_ids.push( + host.app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("the conflicting Action is durable") + .submission_id, + ); + } + + let steps = host + .tick_once() + .expect("the scheduler lawfully classifies the footprint conflict"); + let host = TrustedRuntimeHost::from_parts(host.into_parts()); + assert_eq!(steps.len(), 1); + conflict_submission_id = submission_ids + .into_iter() + .find(|submission_id| { + matches!( + host.echo_operation_action_outcome_v1(submission_id), + Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(_)) + ) + }) + .expect("exactly one Action is rejected by the earlier applied footprint"); + let Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) = + host.echo_operation_action_outcome_v1(&conflict_submission_id) + else { + panic!("the rejected Action retains typed conflict evidence"); + }; + let decision = conflict + .decision_coordinate() + .expect("the terminal conflict names its deciding Tick"); + assert_eq!(decision.writer_head(), head_key); + assert_eq!( + decision.worldline_tick_after(), + steps[0].worldline_tick_after + ); + assert_eq!(decision.commit_id(), steps[0].commit_hash); + assert_eq!(decision.member_index(), 1); + assert_eq!(conflict.blocked_by(), &[0]); + + #[cfg(feature = "host_test")] + { + let mut adversarial = host + .runtime_wal() + .expect("the conflict WAL remains enabled") + .recover_read_only() + .expect("the honest conflict batch recovers"); + adversarial.replace_echo_operation_action_conflict_preparation_for_test( + conflict_submission_id, + digest("forged-conflict-preparation"), + ); + assert!(matches!( + adversarial.validate_echo_operation_action_outcomes_for_test(), + Err(TrustedRuntimeWalError::SchedulerTickBatchMismatch) + )); + let mut false_conflict = host + .runtime_wal() + .expect("the conflict WAL remains enabled") + .recover_read_only() + .expect("the honest conflict batch recovers again"); + false_conflict.replace_echo_operation_action_conflict_blockers_for_test( + conflict_submission_id, + Vec::new(), + ); + assert!(matches!( + false_conflict.validate_echo_operation_parent_states_for_test( + host.runtime(), + host.provenance() + ), + Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { .. }) + )); + } + } + + let (mut recovered, recovered_head, _) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("fresh-host recovery reconstructs the honest footprint conflict"); + let Some(EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict)) = + recovered.echo_operation_action_outcome_v1(&conflict_submission_id) + else { + panic!("fresh-host recovery restores the typed conflict"); + }; + assert_eq!( + conflict + .decision_coordinate() + .expect("recovered conflict retains its deciding Tick") + .writer_head(), + recovered_head + ); +} + +#[test] +fn dormant_head_actions_do_not_consume_runnable_head_admission_capacity() { + let attachment_type = make_type_id("operation-fixture-atom"); + let (mut host, runnable_head, runnable_node, dormant_node) = fixture_host_with_bare_node( + make_node_id("operation-fixture-dormant-node"), + make_type_id("operation-fixture-node"), + Some((attachment_type, b"dormant-before")), + ); + let dormant_head = WriterHeadKey { + worldline_id: runnable_head.worldline_id, + head_id: make_head_id("operation-fixture-dormant-writer"), + }; + let mut parts = host.into_parts(); + parts + .runtime_mut() + .register_writer_head(WriterHead::with_routing( + dormant_head, + PlaybackMode::Play, + InboxPolicy::AcceptAll, + None, + false, + )) + .expect("the dormant sibling head registers"); + host = TrustedRuntimeHost::from_parts(parts); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + + for ordinal in 0..ACTION_BATCH_CANDIDATE_LIMIT_V1 { + let replacement = format!("dormant-after-{ordinal}"); + let invocation = action_invocation( + &host, + &installed, + dormant_head, + dormant_node, + b"dormant-before", + replacement.as_bytes(), + ); + host.app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: dormant_head }, + invocation, + ) + .expect("the dormant invocation is one canonical Action"), + ) + .expect("the dormant Action is accepted"); + } + let mut parts = host.into_parts(); + parts + .runtime_mut() + .set_head_eligibility(dormant_head, HeadEligibility::Dormant) + .expect("the sibling head becomes dormant before admission"); + host = TrustedRuntimeHost::from_parts(parts); + assert!(host + .tick_once() + .expect("dormant Actions do not execute") + .is_empty()); + + let runnable_invocation = action_invocation( + &host, + &installed, + runnable_head, + runnable_node, + b"before", + b"runnable-after", + ); + let runnable_submission_id = host + .app() + .submit_intent( + warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: runnable_head }, + runnable_invocation, + ) + .expect("the runnable invocation is one canonical Action"), + ) + .expect("the runnable Action is accepted") + .submission_id; + + let steps = host + .tick_once() + .expect("the runnable Action is not starved by the dormant head"); + assert_eq!(steps.len(), 1); + assert!(matches!( + host.echo_operation_action_outcome_v1(&runnable_submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + +#[cfg(feature = "host_test")] +#[test] +fn scheduler_wal_failure_rolls_back_and_requeues_action() { + let wal_dir = TempWalDir::new(); + let submission_id; + let head_key; + let node; + + { + let (mut host, fixture_head, fixture_node) = fixture_host(); + head_key = fixture_head; + node = fixture_node; + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the WAL-failure fixture opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"must-rollback", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes one Action"); + submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("acceptance commits before the injected Tick failure") + .submission_id; + host.inject_runtime_wal_filesystem_fault_for_test(FilesystemWalFaultPlan::fail_next( + FilesystemWalFaultTarget::AppendFrame, + )) + .expect("the test injects one scheduler-WAL append failure"); + assert!(host.tick_once().is_err()); + assert!(host + .echo_operation_action_outcome_v1(&submission_id) + .is_none()); + assert!(matches!( + host.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { + ticketed_ingress_id: None, + .. + } + )); + let state = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the rolled-back worldline remains available") + .state(); + assert_eq!(state.current_tick().as_u64(), 0); + assert_eq!( + state + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); + + host.tick_once() + .expect("the same host retries the still-pending Action"); + assert!(matches!( + host.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + assert_eq!( + host.runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the retried worldline remains available") + .state() + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"must-rollback"), + ))) + ); + } + + let (mut recovered, _, _) = fixture_host(); + recovered + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("recovery discards the failed Tick tail and restores the retry"); + assert!(matches!( + recovered.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Decided { .. } + )); + assert!(matches!( + recovered.echo_operation_action_outcome_v1(&submission_id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); +} + +#[cfg(feature = "host_test")] +#[test] +fn scheduler_tick_construction_failure_publishes_no_action_state_or_receipt() { + let (mut host, head_key, node) = fixture_host(); + host.enable_in_memory_runtime_wal() + .expect("the construction-failure fixture WAL opens"); + let installed = install_fixture_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + let state_root_before = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the worldline exists") + .state() + .state_root(); + let invocation = action_invocation( + &host, + &installed, + head_key, + node, + b"before", + b"must-not-commit", + ); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head_key }, + invocation, + ) + .expect("the invocation becomes one Action"); + let submission_id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("acceptance is durable before Tick construction") + .submission_id; + host.inject_echo_operation_action_tick_construction_failure_for_test(); + + let error = host + .tick_once() + .expect_err("the poisoned coordinate prevents Tick construction"); + assert!( + matches!( + &error, + TrustedRuntimeHostError::Runtime(error) + if matches!( + error.as_ref(), + RuntimeError::EchoOperationCommit( + EchoOperationCommitErrorV1::InjectedTickConstructionFailure + ) + ) + ), + "unexpected construction error: {error:?}" + ); + assert!(host + .echo_operation_action_outcome_v1(&submission_id) + .is_none()); + assert!(matches!( + host.runtime().observe_intent_outcome(&submission_id), + warp_core::IntentOutcomeObservation::Pending { .. } + )); + let frontier = host + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("the failed Tick leaves the worldline available"); + assert_eq!(frontier.frontier_tick().as_u64(), 0); + assert_eq!(frontier.state().state_root(), state_root_before); + assert!(frontier.state().tick_history().is_empty()); + assert_eq!( + frontier + .state() + .store(&node.warp_id) + .and_then(|store| store.node_attachment(&node.local_id)), + Some(&AttachmentValue::Atom(AtomPayload::new( + make_type_id("operation-fixture-atom"), + Bytes::from_static(b"before"), + ))) + ); + assert_eq!( + host.runtime_wal() + .expect("the WAL remains enabled") + .commits() + .iter() + .map(|commit| commit.transaction_kind) + .collect::>(), + vec![ + WalTransactionKind::ExecutableOperationInstallation, + WalTransactionKind::SubmissionIntake, + ] + ); +} diff --git a/docs/README.md b/docs/README.md index beec44e4..4562084c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ causal history. Git history is the archive; GitHub owns live work and status. - [Local contract host quickstart](quickstart-local-contract-host.md) - [Echo 1.0 release contract](releases/echo-1.0-contract.md) - [WARP core runtime](spec/warp-core.md) +- [Scheduler WARP core](spec/scheduler-warp-core.md) ## Current Architecture @@ -51,6 +52,10 @@ causal history. Git history is the archive; GitHub owns live work and status. - [Bunny owns reusable geometry](adr/0019-bunny-owns-reusable-geometry.md) - [Retained reading storage and proof boundary](adr/0020-retained-reading-storage-and-proof-boundary.md) - [Public optic and observation boundary](adr/0021-public-optic-observation-boundary.md) +- [Application-requested causal-anchor admission](adr/0022-application-requested-causal-anchor-admission.md) +- [Admitted executable-operation packages](adr/0023-admitted-executable-operation-packages.md) +- [Anchored-node creation from absence](adr/0024-anchored-node-creation-from-absence.md) +- [Scheduler-owned executable-operation Actions](adr/0025-scheduler-owned-executable-operation-actions.md) ## Normative Contracts diff --git a/docs/adr/0023-admitted-executable-operation-packages.md b/docs/adr/0023-admitted-executable-operation-packages.md index cffb21fa..77986abd 100644 --- a/docs/adr/0023-admitted-executable-operation-packages.md +++ b/docs/adr/0023-admitted-executable-operation-packages.md @@ -6,6 +6,16 @@ - **Status:** Accepted - **Date:** 2026-07-18 - **Partially supersedes:** ADR 0015 for newly authored executable operations +- **Amended by:** ADR 0025 for scheduler-owned Action execution + +> **Scheduler lifecycle amendment:** This ADR established exact package, +> invocation, private-evaluation, and recovery semantics. ADR 0025 supersedes +> direct prepare/commit as the application lifecycle: canonical executable +> Actions now enter ordinary durable ingress, and only the scheduler invokes +> private evaluation while constructing an atomic Tick. The direct methods +> remain documentation-hidden public `TrustedRuntimeHost` compatibility/test +> seams; they are absent from `TrustedRuntimeApp`, and their eventual removal is +> tracked by [issue #689](https://github.com/flyingrobots/echo/issues/689). ## Context diff --git a/docs/adr/0025-scheduler-owned-executable-operation-actions.md b/docs/adr/0025-scheduler-owned-executable-operation-actions.md new file mode 100644 index 00000000..0a5e0414 --- /dev/null +++ b/docs/adr/0025-scheduler-owned-executable-operation-actions.md @@ -0,0 +1,205 @@ + + + +# ADR 0025: Scheduler-Owned Executable-Operation Actions + +- **Status:** Accepted +- **Date:** 2026-07-24 +- **Amends:** ADR 0023 + +## Context + +ADR 0023 introduced Echo-interpreted executable-operation packages so +application semantics no longer need a native callback. Its first runtime +crossing nevertheless exposes a second execution lifecycle: + +```text +admit invocation +prepare privately +commit prepared operation directly +``` + +That crossing proved exact package resolution, bounded private evaluation, +typed obstruction, patch validation, and durable recovery. It is not the +permanent application route. Echo's public write lifecycle is an `Action` +accepted into causal ingress, selected by the scheduler, and decided as part of +one atomic `Tick`. + +Keeping direct operation commit as a peer lifecycle would let application or +host-adapter code choose when evaluation runs, bypass normal Lane/head +selection, and publish a singleton consequence outside the scheduler's +multi-Action decision. That would preserve the same callback-shaped authority +problem Edict was introduced to remove, merely behind a different artifact. + +The current scheduler and WAL also assume different cardinalities: + +- one head commit may decide several ingress envelopes; +- `TrustedRuntimeHost::tick_once` writes one scheduler-WAL transaction per + receipt correlation; +- the filesystem WAL rejects more than one such transaction because the group + would not be failure-atomic. + +One Tick must be one durable transaction, regardless of its Action count. + +## Decision + +### Executable operations enter through canonical Actions + +Echo reserves one stable `IntentKind` for executable-operation Actions. Its +payload is exactly one canonical `EchoOperationInvocationV1`; package meaning +is never copied into the envelope and no native callback is attached. + +Application-facing code may only submit that envelope through the ordinary +submission boundary. An acknowledgement means the canonical Action and its +acceptance evidence have committed to the WAL. It does not mean the Action has +executed. + +The runtime owner configures invocation-admission policy. After acceptance, +Echo resolves the exact installed package, issues ticketed runtime ingress, and +places the Action in the normal target head inbox. A restart reconstructs the +accepted pending Action from retained submission material; admission and +staging are deterministic and repeatable. + +### The scheduler owns evaluation + +Only scheduler Tick construction invokes the private bounded evaluator for an +Action. The evaluator remains private and pure with respect to the parent +worldline: + +- success returns one complete prepared candidate; +- obstruction returns typed evidence and no parent-visible operations; +- neither result mutates the parent state. + +The direct `prepare_echo_operation_v1` and +`commit_prepared_echo_operation_v1` methods remain publicly reachable on +`TrustedRuntimeHost` as explicitly transitional, documentation-hidden +compatibility/test seams. `TrustedRuntimeApp` does not expose them, but that +application-facing type boundary is not a claim that Rust visibility prevents +every downstream host owner from calling the methods. They are not the +application architecture. Their migration and removal are tracked by +[issue #689](https://github.com/flyingrobots/echo/issues/689). + +### Legacy and executable ingress never share an evaluator batch + +Provider/native ingress and executable-operation Actions are different +execution categories. A head inbox may contain both while migrations are in +flight, but one Tick candidate batch is homogeneous. + +When both categories are pending, the exact parent global-Tick parity selects +the category: an even scheduler round selects executable Actions and an odd +round selects provider/native ingress. When only one category is pending, that +category proceeds immediately. Entries remain canonically ordered by ingress +identity inside the selected category and existing inbox budgets still apply. +The durable global coordinate advances once per scheduler pass, so every head +alternates even when several heads advance one shared worldline. This choice +prevents caller-controlled hashes from starving one evaluator category, remains +stable across restart, and makes it impossible for an executable Action to fall +through to a native callback engine. + +### One exact parent basis, per-Action application propositions + +Actions selected into one Tick share the exact Echo parent coordinate: + +- writer head; +- worldline tick; +- committing global-tick predecessor; +- state root; +- parent commit identity. + +Each Action retains its own application-basis proposition. Two independent +Actions over different domain values therefore need not have byte-identical +complete `EchoOperationEvaluationBasisV1` values. This amends ADR 0023's +informal statement that composed preparations share identical complete basis +bytes: they share the runtime parent fields, while the separately typed +application proposition remains candidate-specific. + +### Tick construction is atomic + +Prepared candidates are considered in canonical Action order. The scheduler +reserves their actual footprints against already accepted candidates: + +- independent candidates are applied to a private successor state; +- footprint conflicts produce a rejected Action outcome naming earlier + applied blockers; +- evaluator obstructions produce a typed obstructed Action outcome with no + blockers and no mutation. + +All accepted candidate operations are canonicalized into one +`WarpTickPatchV1`. Tick construction produces one snapshot, one Tick receipt +with one entry per Action, one provenance entry, and one worldline frontier +advance. Failure while evaluating, composing, applying, or validating discards +the private successor in full. + +The generic Tick receipt classifies scheduler disposition. A separate typed +executable-Action outcome binds the exact invocation, preparation or +obstruction and Tick identity. Applied Actions additionally bind their +committed patch member; obstruction and footprint-conflict outcomes explicitly +carry no committed consequence. Application semantics are not inferred from a +generic receipt label. Rejected outcomes retain the deciding writer head, +worldline and global Tick coordinates, commit and Tick-receipt identities, and +their canonical member index, so their evidence is interpretable without a +separate correlation lookup. + +### One Tick is one WAL transaction + +A scheduler Tick WAL transaction contains, in canonical Action order: + +```text +TickReceiptRecorded +ReceiptCorrelationRecorded + ActionOutcomeRecorded +... one correlation/outcome pair per Action ... +RuntimeStateDeltaRecorded +``` + +The single batched Tick receipt record retains every per-Action scheduler +decision and exact causal receipt coordinate. Each following correlation and +typed outcome names its corresponding Action. The state delta appears exactly +once because the Tick has exactly one atomic state consequence. + +The WAL transaction is committed before the live runtime, frontier, provenance, +Action outcomes, or receipts are published. On append failure, live state is +restored to the accepted-pending posture. Recovery validates the whole group, +reconstructs the state transition once, and restores every per-Action outcome +and receipt correlation. + +## Consequences + +- The application-facing `TrustedRuntimeApp` cannot invoke executable + evaluation or direct commit. Downstream code that owns a + `TrustedRuntimeHost` can still reach the documentation-hidden transitional + methods until issue #689 removes that exception. +- Two independent executable-operation Actions can contribute to one + scheduler-owned Tick. +- A typed obstruction is durable evidence and cannot hide a state mutation. +- Filesystem durability no longer depends on several scheduler transactions + being atomically appended as an external batch. +- Existing executable package, program, invocation, and direct singleton + receipt bytes remain unchanged. +- Native/provider callback ingress remains compatibility infrastructure and is + not extended by this decision. + +## Acceptance + +The implementation is accepted only with executable witnesses that prove: + +1. Action acknowledgement follows WAL commit. +2. Accepted pre-Tick Actions survive restart as pending work. +3. Submission and runtime admission do not evaluate or mutate state. +4. Only scheduler Tick construction invokes private evaluation. +5. Two independent Actions produce one Tick and one atomic state consequence. +6. Tick construction or WAL failure publishes no partial state or receipt. +7. Typed obstruction contributes no operations. +8. The decided Tick is durable before frontier and receipt publication. +9. Fresh-host recovery reconstructs Action outcomes, Tick, state, and receipts. +10. Direct prepare/commit is marked transitional and absent from the + application-facing surface. +11. Removal of the remaining public host-owner compatibility seams is tracked + separately rather than claimed complete by this decision. + +## Non-Goals + +- Real Edict compiler emission or a production Graft lawpack. +- Provider-v1 or native callback generalization. +- Arbitrary multi-record program semantics. +- Cross-head or cross-worldline atomic Ticks. +- Migration from Graft's git-warp database. diff --git a/docs/adr/README.md b/docs/adr/README.md index 170b0ecf..4ca1995d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -46,6 +46,7 @@ track work, progress, priority, or release readiness. | [0022](0022-application-requested-causal-anchor-admission.md) | Accepted | Application-requested, Echo-owned anchor admission | | [0023](0023-admitted-executable-operation-packages.md) | Accepted | Admitted executable operation packages | | [0024](0024-anchored-node-creation-from-absence.md) | Accepted | Anchored-node creation as a separate executable program | +| [0025](0025-scheduler-owned-executable-operation-actions.md) | Accepted | Scheduler-owned executable-operation Actions | ADR 0006 predates this index contract and did not declare a status. Its superseded tombstone preserves that fact without silently ratifying the old diff --git a/docs/architecture/application-contract-hosting.md b/docs/architecture/application-contract-hosting.md index 9824987d..4770ab0f 100644 --- a/docs/architecture/application-contract-hosting.md +++ b/docs/architecture/application-contract-hosting.md @@ -151,23 +151,28 @@ accepts a distinct executable-operation category for new application mutations. Echo now implements two bounded anchored-attachment programs in that runtime slice: the original update-only compare-and-set and a separately identified single-node-plus-alpha-attachment create-if-absent profile. Package -and invocation admission, private bounded evaluation, exact-basis singleton -commit, typed evidence, WAL, and callback-free fresh-host recovery validate the -selected program's exact footprint, budget, result, and patch shape. -Operations targeting a descended WARP retain and meter every portal attachment -in the root-to-target reachability chain as a causal input. -Noncommitted evidence is returned to the caller but does not enter the -operation-tick WAL. The slice exposes no application matcher, executor, or -footprint callback. It is not yet emitted by Edict, does not implement Jedit -`ReplaceRange` or Graft's multi-record mutation, and has no structurally -separate target verifier or general scheduler batch composition. Provider v1 -remains stable while consumers migrate; it is not renamed or silently -reinterpreted as the executable-operation corridor. The program digest -supplies executable meaning only: it cannot independently confer an operation -coordinate, invocability, or authority, and Echo cannot install or invoke it -naked. The admitted operation package binds the public contract and semantic -closure to the exact program, after which Echo independently admits each -invocation. +and invocation admission, WAL-durable Action intake, ordinary head-inbox +staging, scheduler-owned bounded evaluation, exact-basis multi-Action Tick +composition, typed per-Action outcomes, and callback-free fresh-host recovery +validate each selected program's exact footprint, budget, result, and patch +shape. Operations targeting a descended WARP retain and meter every portal +attachment in the root-to-target reachability chain as a causal input. +Scheduler-selected obstructions and footprint conflicts enter the decided Tick +as typed no-hidden-mutation outcomes. The decided Tick reaches the WAL before +state, frontier, or Receipt publication. The transitional direct +prepare/commit seam can still return noncommitted evidence to trusted host +tests, but it is not the application lifecycle. + +The slice exposes no application matcher, executor, or footprint callback. It +is not yet emitted by Edict, does not implement Jedit `ReplaceRange` or any +Graft operation, has no structurally separate target verifier, and does not +claim cross-head atomic filesystem-WAL persistence. Provider v1 remains stable +while consumers migrate; it is not renamed or silently reinterpreted as the +executable-operation corridor. The program digest supplies executable meaning +only: it cannot independently confer an operation coordinate, invocability, or +authority, and Echo cannot install or invoke it naked. The admitted operation +package binds the public contract and semantic closure to the exact program, +after which Echo independently admits each invocation. ## External Edict Provider Artifacts diff --git a/docs/spec/scheduler-warp-core.md b/docs/spec/scheduler-warp-core.md index 80b4046d..b2471b03 100644 --- a/docs/spec/scheduler-warp-core.md +++ b/docs/spec/scheduler-warp-core.md @@ -10,6 +10,7 @@ Depends on: - [Canonical Inbox Sequencing](canonical-inbox-sequencing.md) - [SPEC-0003 - DPO Concurrency Litmus v0](SPEC-0003-dpo-concurrency-litmus-v0.md) - [WARP Tick Patch](warp-tick-patch.md) +- [ADR 0025 - Scheduler-Owned Executable-Operation Actions](../adr/0025-scheduler-owned-executable-operation-actions.md) ## Purpose @@ -36,10 +37,12 @@ Reservation checks all candidate resources for conflict and marks resources only A candidate conflicts when it writes a resource another admitted candidate reads or writes, or when its boundary port claim overlaps another admitted boundary port claim. Reads may overlap reads. The broad WARP outcome algebra is `Derived | Plural | Conflict | -Obstruction`. Echo's local `TickReceipt` entries currently realize the narrower -tick-scale shape `Applied / Rejected(FootprintConflict)`, with blocker -attribution for conflicts. `Plural` belongs to broader braid/replica-scale work -until an executable local claim requires it. +Obstruction`. Echo's local `TickReceipt` entries realize the narrower +tick-scale shape `Applied / Rejected(FootprintConflict | +ExecutableOperationObstruction)`. Footprint conflicts name earlier applied +blockers. An executable-operation obstruction has no blockers and contributes +no mutation. `Plural` belongs to broader braid/replica-scale work until an +executable local claim requires it. Conflict rejection is final for that tick attempt. Retry is a new explicit causal act, not a hidden retry queue. @@ -56,3 +59,24 @@ Pending rewrites drain in lexicographic order derived from `scope_hash`, stable ## Decision 5: Radix reservation is the default implementation The default scheduler uses generation-stamped active sets for membership checks. Expected reservation cost is proportional to candidate footprint size. + +## Decision 6: Executable Actions are evaluated only inside Tick construction + +A canonical executable-operation Action enters the same durable submission and +head-inbox lifecycle as other work. The runtime resolves the exact installed +package and admission policy, but neither submission nor admission evaluates +application semantics. + +The scheduler partitions executable and legacy/native ingress deterministically +so the two evaluator categories never share one candidate batch. When both are +pending, exact parent global-Tick parity alternates the selected category for +every head on each scheduler pass; when only one is pending, it proceeds +immediately. Inside either category, ingress order is canonical. Private bounded +evaluation yields either a complete prepared candidate or a typed no-mutation +obstruction. The scheduler reserves successful candidate footprints, constructs +one composite consequence, and emits one Tick receipt entry per Action. + +The successor state remains private until the complete Tick transaction is +durable. Construction failure discards it. WAL failure restores the +accepted-pending posture. The direct executable-operation prepare/commit methods +are transitional host/test seams, not a second application execution path. diff --git a/docs/topics/GeneratedRules.md b/docs/topics/GeneratedRules.md index 67175756..93ac29e4 100644 --- a/docs/topics/GeneratedRules.md +++ b/docs/topics/GeneratedRules.md @@ -140,11 +140,13 @@ exact canonical ExecutableOperationPackageV1 bytes -> Echo-owned package admission under a separate policy -> installed data-only EchoOperationProgramV1 -> exact basis-bearing canonical invocation +-> canonical Action submission retained before acknowledgement -> Echo-owned invocation admission under authority and delegated budget --> bounded private evaluation with recorded actual footprint --> exact-basis singleton commit attempt --> one committed TickPatch and typed receipt, or returned typed no-patch evidence --> committed outcomes only: execution-kernel WAL retention and callback-free recovery +-> ordinary head-inbox staging and scheduler selection at one exact basis +-> bounded private evaluation with recorded actual footprint during Tick construction +-> deterministic composition of independent Actions into one Tick consequence +-> typed committed, obstructed, or footprint-conflict outcome for every Action +-> decided Tick WAL retention before publication and callback-free recovery ``` The first two programs are an anchored typed-node alpha-attachment @@ -176,15 +178,18 @@ invocability. The subordinate program supplies executable meaning only. Echo cannot install or invoke a naked program digest, and a generated artifact does not confer caller authority. Provider v1 remains stable while existing consumers migrate; it is not silently reinterpreted as this new category. -The program bytes bind their interpreter and intrinsic profiles directly. The -parent patch and singleton tick entry use the admitted installed-operation -identity as their rule-pack/rule identity, so two packages that reuse identical -program bytes do not collapse into one causal operation identity. -The current slice composes one preparation and does not claim general scheduler -batch integration, Graft-style multi-record mutation, or independent -implementation evidence. Its canonical decoding, identity recomputation, -budget and footprint checks, exact-basis comparison, deterministic -repeatability, and WAL recovery are deterministic self-validation. +The program bytes bind their interpreter and intrinsic profiles directly. +Direct compatibility commits retain their singleton evidence, while +scheduler-owned Action Ticks use the admitted installed-operation identity for +each receipt-entry rule identity and bind every applied preparation into one +canonical composite consequence. Two packages that reuse identical program +bytes therefore do not collapse into one causal operation identity. +The current slice composes multiple independent, same-basis executable Actions +inside one scheduler-owned Tick. It does not claim cross-category scheduler +composition, Graft-style multi-record mutation, or independent implementation +evidence. Its canonical decoding, identity recomputation, budget and footprint +checks, exact-basis comparison, deterministic repeatability, and WAL recovery +are deterministic self-validation. For descended targets, the retained footprint and patch inputs include every portal attachment in the validated root-to-target reachability chain. diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index 5a1348c9..34eaf1b4 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -17,7 +17,7 @@ Echo may only claim what its WAL can recover. ## What We Found -The current runtime WAL evidence says nine concrete things. +The current runtime WAL evidence says ten concrete things. First, accepted-submission evidence is not just an in-memory editor event. The WAL-backed ACK path, `submit_intent_with_runtime_wal_ack(...)`, returns only @@ -89,9 +89,14 @@ state-delta record is inspectable but obstructs writable startup. Transaction construction also requires the receipt and correlation to name the same causal event and the retained state delta to bind that receipt's content commitment before the transaction can commit. -Each scheduler transaction must also contain exactly one receipt, correlation, -and runtime state-delta frame; recovery rejects duplicate claims rather than -selecting whichever frame appears first. +Legacy singleton scheduler transactions contain exactly one receipt, +correlation, and runtime state-delta frame. Scheduler-owned +executable-operation Action Ticks instead retain exactly one batched Tick +receipt record containing every per-Action decision, one ordered correlation +and typed Action-outcome pair per selected Action, and exactly one runtime state +delta for the whole Tick. Recovery rejects missing, duplicated, reordered, +mixed, or cross-Tick claims rather than selecting whichever frame appears +first. WAL activation is non-lossy: if the live host contains submissions, staged ingress, receipt correlations, provenance, pending inbox work, cycle progress, or worldline state that recovered WAL evidence cannot reproduce, activation @@ -127,6 +132,22 @@ or mis-scoped frame/frontier shapes. These records do not make a naked program digest invocable and do not persist the process-local authority needed to commit an unretained preparation. +Tenth, canonical executable-operation Actions use the ordinary WAL-backed +submission acknowledgement before execution. A crash after acknowledgement and +before scheduler selection restores accepted pending work. Runtime-owned +admission resolves the exact installed package, but only scheduler Tick +construction invokes private bounded evaluation. Two independent Actions can +produce one composite Tick with one snapshot, one provenance entry, one +frontier advance, per-Action typed outcomes, and one replayable state delta. + +The scheduler transaction commits before state, frontier, receipt, or Action +outcome publication. Construction and WAL failures publish none of them. A +typed evaluator obstruction is retained as a lawful no-mutation Tick entry. +Fresh-host recovery binds each Action outcome back to its exact retained +envelope, invocation identity, installed operation, scope, Tick entry, +disposition, blocker set, composite patch, and final state root. Swapped or +misclassified outcome evidence therefore fails closed. + ## Boundaries The WAL belongs to the trusted runtime host. Application-facing code can submit @@ -288,12 +309,26 @@ canonical package and invocation admission, authority and budget refusal, exact-basis noncommit, program-substitution refusal, same-host preparation authority, typed private-evaluation obstruction, deterministic duplicate-host results, append-only WAL codes, activation refusal for process-only -installations, and fresh-host installation and consequence recovery. +installations, accepted-Action recovery, scheduler-owned decided Ticks, and +fresh-host installation and consequence recovery. + +The Action/Tick witnesses to read first are: + +- `accepted_executable_action_recovers_pending_before_scheduler_evaluation` +- `scheduler_commits_two_independent_executable_actions_in_one_durable_tick` +- `scheduler_tick_construction_failure_publishes_no_action_state_or_receipt` +- `scheduler_wal_failure_rolls_back_and_requeues_action` +- `typed_action_obstruction_is_durable_and_contributes_no_mutation` +- `filesystem_wal_recovers_installed_meaning_consequence_and_typed_receipt` The host surface lives in `crates/warp-core/src/trusted_runtime_host.rs`, especially `TrustedRuntimeHost`, `TrustedRuntimeApp`, `TrustedRuntimeWal`, `submit_intent_with_runtime_wal_ack(...)`, `admit_causal_anchor(...)`, `causal_anchor_by_id(...)`, and `recover_read_only()`. +`TrustedRuntimeHostParts` makes host decomposition identity-preserving: +`into_parts()` and `from_parts(...)` carry WAL, authority, policy, pending +admission, and typed Action outcomes together with runtime, provenance, and +engine ownership. Related current authority lives in `docs/topics/RuntimeAuthority.md`, `docs/architecture/continuum-transport.md`, and diff --git a/scripts/verify-local.sh b/scripts/verify-local.sh index 89c6fcf0..ef8fc503 100755 --- a/scripts/verify-local.sh +++ b/scripts/verify-local.sh @@ -1157,7 +1157,9 @@ pre_push_feature_string_for_test_target() { printf '%s\n' "native_rule_bootstrap,host_test" ;; warp-core:external_consumer_contract_fixture_tests|\ - warp-core:provider_contract_admission_tests|\ + warp-core:provider_contract_admission_tests) + printf '%s\n' "native_rule_bootstrap,trusted_runtime" + ;; warp-core:executable_operation_pipeline_tests) printf '%s\n' "native_rule_bootstrap,trusted_runtime" ;; @@ -1188,6 +1190,17 @@ pre_push_feature_string_for_test_target() { esac } +pre_push_extra_feature_strings_for_test_target() { + local crate="$1" + local test_target="$2" + + case "${crate}:${test_target}" in + warp-core:executable_operation_pipeline_tests) + printf '%s\n' "native_rule_bootstrap,trusted_runtime,host_test" + ;; + esac +} + append_pre_push_rust_slice() { local slice="$1" local slices_name="$2" @@ -1196,7 +1209,7 @@ append_pre_push_rust_slice() { collect_pre_push_rust_slices() { local -a slices=() - local file crate target filter features relative_test + local file crate target filter features extra_features relative_test while IFS= read -r file; do [[ -z "$file" ]] && continue @@ -1211,6 +1224,12 @@ collect_pre_push_rust_slices() { target="$(basename "$file" .rs)" features="$(pre_push_feature_string_for_test_target "$crate" "$target")" append_pre_push_rust_slice "${crate}|test|${target}|${features}|" slices + while IFS= read -r extra_features; do + [[ -z "$extra_features" ]] && continue + append_pre_push_rust_slice \ + "${crate}|test|${target}|${extra_features}|" \ + slices + done < <(pre_push_extra_feature_strings_for_test_target "$crate" "$target") ;; crates/*/src/lib.rs) crate="$(printf '%s\n' "$file" | sed -n 's#^crates/\([^/]*\)/.*#\1#p')" @@ -1430,17 +1449,31 @@ warp_core_feature_args_for_test() { run_warp_core_test_target() { local lane="$1" local test_target="$2" + local extra_features local -a feature_args=() mapfile -t feature_args < <(warp_core_feature_args_for_test "$test_target") lane_cargo "$lane" test -p warp-core "${feature_args[@]}" --test "$test_target" + while IFS= read -r extra_features; do + [[ -z "$extra_features" ]] && continue + lane_cargo "$lane" test -p warp-core \ + --features "$extra_features" \ + --test "$test_target" + done < <(pre_push_extra_feature_strings_for_test_target "warp-core" "$test_target") } run_warp_core_clippy_test_target() { local lane="$1" local test_target="$2" + local extra_features local -a feature_args=() mapfile -t feature_args < <(warp_core_feature_args_for_test "$test_target") lane_cargo "$lane" clippy -p warp-core "${feature_args[@]}" --test "$test_target" -- -D warnings -D missing_docs + while IFS= read -r extra_features; do + [[ -z "$extra_features" ]] && continue + lane_cargo "$lane" clippy -p warp-core \ + --features "$extra_features" \ + --test "$test_target" -- -D warnings -D missing_docs + done < <(pre_push_extra_feature_strings_for_test_target "warp-core" "$test_target") } prepare_warp_wasm_scope() { diff --git a/tests/docs/test_adr_namespace.sh b/tests/docs/test_adr_namespace.sh index 45417e27..d0b07989 100755 --- a/tests/docs/test_adr_namespace.sh +++ b/tests/docs/test_adr_namespace.sh @@ -35,9 +35,10 @@ readonly current_adrs=( "0022-application-requested-causal-anchor-admission.md" "0023-admitted-executable-operation-packages.md" "0024-anchored-node-creation-from-absence.md" + "0025-scheduler-owned-executable-operation-actions.md" ) -readonly current_adr_last=24 +readonly current_adr_last=25 readonly superseded_legacy_adrs=( "ADR-0003-Materialization-Bus.md" diff --git a/tests/hooks/test_verify_local.sh b/tests/hooks/test_verify_local.sh index 517b4135..feccd77c 100755 --- a/tests/hooks/test_verify_local.sh +++ b/tests/hooks/test_verify_local.sh @@ -1611,6 +1611,15 @@ else printf '%s\n' "$fake_pre_push_provider_contract_output" fi +fake_pre_push_executable_operation_output="$(run_fake_verify pre-push crates/warp-core/tests/executable_operation_pipeline_tests.rs)" +fake_pre_push_executable_operation_cargo_log="$(extract_log_section cargo-log "$fake_pre_push_executable_operation_output")" +if printf '%s\n' "$fake_pre_push_executable_operation_cargo_log" | grep -q -- 'test -p warp-core --features native_rule_bootstrap,trusted_runtime,host_test --test executable_operation_pipeline_tests'; then + pass "pre-push keeps required scheduler Action features for the exact integration test" +else + fail "pre-push should keep required scheduler Action features for executable_operation_pipeline_tests" + printf '%s\n' "$fake_pre_push_executable_operation_output" +fi + fake_pre_push_installed_contract_output="$(run_fake_verify pre-push crates/warp-core/tests/installed_contract_registry_tests.rs)" fake_pre_push_installed_contract_cargo_log="$(extract_log_section cargo-log "$fake_pre_push_installed_contract_output")" if printf '%s\n' "$fake_pre_push_installed_contract_cargo_log" | grep -q -- 'test -p warp-core --features native_rule_bootstrap --test installed_contract_registry_tests'; then diff --git a/xtask/src/hello_echo.rs b/xtask/src/hello_echo.rs index b341b913..bb6eb11b 100644 --- a/xtask/src/hello_echo.rs +++ b/xtask/src/hello_echo.rs @@ -679,6 +679,12 @@ fn receipt_report(receipt: &TickReceipt) -> ReceiptReport { conflict_count += 1; "rejected:footprint-conflict" } + TickReceiptDisposition::Rejected( + TickReceiptRejection::ExecutableOperationObstruction, + ) => { + rejected_count += 1; + "rejected:executable-operation-obstruction" + } }; entries.push(ReceiptEntryReport { index,