diff --git a/Cargo.lock b/Cargo.lock index 42983ba38..c1a7885e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2164,6 +2164,7 @@ dependencies = [ "graphforge-core", "graphforge-cypher", "graphforge-exec", + "graphforge-filesystem", "graphforge-io", "graphforge-ir", "graphforge-knowledge", diff --git a/Makefile b/Makefile index c7af7ca62..bd0e4135a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help lint format type-check security workflow-lint license-check third-party-notices third-party-notices-check cargo-deny-licenses test pre-push pre-push-clean pre-push-preflight pre-push-fast bazel-test clean test-tck docstring-coverage test-network benchmark test-perf test-perf-xs test-perf-slow test-perf-large coverage coverage-rust coverage-python coverage-node coverage-quick coverage-report coverage-diff coverage-strict check-coverage check-coverage-rust check-coverage-python check-coverage-node check-patch-coverage test-durations test-analytics docs-serve docs-build docs-clean cargo-build codspeed-build codspeed-build-walltime codspeed-run bench-traversal bench-fixed-hop-limit bench-fixed-hop-livejournal bench-m4-entry bench-g500-scale20 bench-g500-ladder bench-adjacency-200m bench-file-backed-128m m4-entry-matrix-check durability-isolation-check native-consumers release-load-matrix-check release-load-matrix bulk-construction-conformance-check bulk-construction-conformance cargo-test cargo-check cargo-clippy cargo-fmt cargo-fmt-check clean-builds clean-builds-all pnpm-install pnpm-build pnpm-test-bdd install build release-version-check package-license-verify publish-dry-run publish-dry-run-npm publish-dry-run-docs publish-dry-run-python publish-dry-run-cargo record-release-artifacts clean-env-verify-check clean-env-verify-preflight clean-env-verify +.PHONY: help lint format type-check security workflow-lint license-check third-party-notices third-party-notices-check cargo-deny-licenses test pre-push pre-push-clean pre-push-preflight pre-push-fast bazel-test clean test-tck docstring-coverage test-network benchmark test-perf test-perf-xs test-perf-slow test-perf-large coverage coverage-rust coverage-python coverage-node coverage-quick coverage-report coverage-diff coverage-strict check-coverage check-coverage-rust check-coverage-python check-coverage-node check-patch-coverage test-durations test-analytics docs-serve docs-build docs-clean cargo-build codspeed-build codspeed-build-walltime codspeed-run bench-traversal bench-fixed-hop-limit bench-fixed-hop-livejournal bench-m4-entry bench-g500-scale20 bench-g500-ladder g500-ladder-qualification g500-ladder-qualification-check bench-adjacency-200m bench-file-backed-128m m4-entry-matrix-check durability-isolation-check native-consumers release-load-matrix-check release-load-matrix bulk-construction-conformance-check bulk-construction-conformance cargo-test cargo-check cargo-clippy cargo-fmt cargo-fmt-check clean-builds clean-builds-all pnpm-install pnpm-build pnpm-test-bdd install build release-version-check package-license-verify publish-dry-run publish-dry-run-npm publish-dry-run-docs publish-dry-run-python publish-dry-run-cargo record-release-artifacts clean-env-verify-check clean-env-verify-preflight clean-env-verify help: ## Show this help message @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' @@ -329,6 +329,19 @@ bench-g500-ladder: ## Bounded billion-edge scale ladder S20-S26 first-fail evid GF_G500_LADDER_MAX_SCALE="$$GF_G500_LADDER_MAX_SCALE" \ cargo test -p graphforge-api --release --test scale_g500_ladder ladder_public_facade_first_fail_evidence -- --ignored --nocapture --test-threads=1 +g500-ladder-qualification-check: ## Validate #951 disk attribution and conservative S26 projection + @test -n "$$EVIDENCE" || (echo "EVIDENCE is required" && exit 2) + uv run --frozen --with jsonschema python scripts/ci/validate-g500-ladder-qualification.py "$$EVIDENCE" + +g500-ladder-qualification: ## Build and validate #951 evidence from adjacent real certifications + @test -n "$$LOW_CERT" || (echo "LOW_CERT is required" && exit 2) + @test -n "$$HIGH_CERT" || (echo "HIGH_CERT is required" && exit 2) + @test -n "$$EVIDENCE" || (echo "EVIDENCE is required" && exit 2) + @test -n "$$VOLUME_BYTES" || (echo "VOLUME_BYTES is required" && exit 2) + @test -n "$$RESERVED_HEADROOM_BYTES" || (echo "RESERVED_HEADROOM_BYTES is required" && exit 2) + uv run --frozen python scripts/ci/build-g500-ladder-qualification.py "$$LOW_CERT" "$$HIGH_CERT" "$$EVIDENCE" --volume-bytes "$$VOLUME_BYTES" --reserved-headroom-bytes "$$RESERVED_HEADROOM_BYTES" + $(MAKE) g500-ladder-qualification-check EVIDENCE="$$EVIDENCE" + bench-adjacency-200m: ## >200M-edge public adjacency build evidence (#336; ignored, scale-host) GF_ADJACENCY_SCALE_EVIDENCE_OUT="$(CURDIR)/docs/development/adjacency-200m-evidence.json" \ GF_ADJACENCY_SCALE_WORK="$(CURDIR)/build/adjacency-200m-work" \ diff --git a/crates/graphforge-api/BUILD.bazel b/crates/graphforge-api/BUILD.bazel index 9243ae160..ec78b2640 100644 --- a/crates/graphforge-api/BUILD.bazel +++ b/crates/graphforge-api/BUILD.bazel @@ -210,7 +210,7 @@ gf_rust_integration_test( data = _API_TEST_DATA, size = "large", timeout = "long", - deps = _API_DEPS, + deps = _API_DEPS + ["//crates/graphforge-filesystem:graphforge_filesystem"], ) gf_rust_integration_test( diff --git a/crates/graphforge-api/Cargo.toml b/crates/graphforge-api/Cargo.toml index f212b44e0..401e456d0 100644 --- a/crates/graphforge-api/Cargo.toml +++ b/crates/graphforge-api/Cargo.toml @@ -39,6 +39,7 @@ cucumber = { workspace = true } tokio = { workspace = true } graphforge-cypher = { path = "../graphforge-cypher" } graphforge-storage = { path = "../graphforge-storage", features = ["test-failpoints", "test-support"] } +graphforge-filesystem = { path = "../graphforge-filesystem" } # BDD runner for the public-API + TCK feature files (tests/features/). # A custom-harness target (cucumber drives its own main). diff --git a/crates/graphforge-api/src/lib.rs b/crates/graphforge-api/src/lib.rs index b07bc67ff..b3ef258f7 100644 --- a/crates/graphforge-api/src/lib.rs +++ b/crates/graphforge-api/src/lib.rs @@ -547,6 +547,22 @@ impl std::fmt::Debug for GraphForge { } impl GraphForge { + /// Capture authenticated logical and physical storage attribution for the + /// generation visible to this facade. + /// + /// The storage layer walks only the generation's authenticated inventories + /// and opens retained file capabilities. It never recursively scans the + /// project directory, and qualification fails closed on an unclassified + /// graph artifact. + pub fn storage_attribution( + &self, + ) -> Result { + let generation = self.generation_for_read()?; + let snapshot = graphforge_storage::capture_storage_attribution(&generation)?; + snapshot.validate_for_qualification()?; + Ok(snapshot) + } + pub(crate) fn stage_project_generation( &self, request: &graphforge_storage::ProjectGenerationRequest, @@ -4099,6 +4115,19 @@ fn materialize_compact_graph_target( files_opened_in_place: 0, files_reused: reused.files_reused, bytes_reused: reused.bytes_reused, + application_read_bytes: reused + .application_read_bytes + .saturating_add(copied.application_read_bytes), + application_read_calls: reused + .application_read_calls + .saturating_add(copied.application_read_calls), + application_write_bytes: reused + .application_write_bytes + .saturating_add(copied.application_write_bytes), + application_write_calls: reused + .application_write_calls + .saturating_add(copied.application_write_calls), + fsync_calls: reused.fsync_calls.saturating_add(copied.fsync_calls), }; Ok(evidence) } diff --git a/crates/graphforge-api/src/portable.rs b/crates/graphforge-api/src/portable.rs index 3da4bfe06..fb4fc31dd 100644 --- a/crates/graphforge-api/src/portable.rs +++ b/crates/graphforge-api/src/portable.rs @@ -60,6 +60,20 @@ pub struct PortableV2ImportResult { pub generation_uuid: Uuid, /// Whether the operation replayed an identical publication. pub idempotent_replay: bool, + /// Exact private-materialization identity allocation for lifecycle qualification. + #[doc(hidden)] + pub materialized_identity_allocated_bytes: std::collections::BTreeMap, + /// Exact published-project identity union for lifecycle qualification, + /// including controls and every retained generation. + #[doc(hidden)] + pub published_identity_allocated_bytes: std::collections::BTreeMap, + /// Exact identities durably removed by private-materialization cleanup. + #[doc(hidden)] + pub materialized_cleanup_removed_identity_allocated_bytes: + std::collections::BTreeMap, + /// Whether the cleanup namespace synchronization completed. + #[doc(hidden)] + pub materialized_cleanup_parent_sync_confirmed: bool, } /// Publish a verified local portable-v2 package to an OCI registry. @@ -186,6 +200,18 @@ pub struct PortableV2ExportFacadeResult { pub selection_fingerprint: String, /// Caller-selected output path. pub output: PathBuf, + /// Exact writer-owned published allocation for lifecycle qualification. + #[doc(hidden)] + #[serde(skip)] + pub allocation_identity_allocated_bytes: std::collections::BTreeMap, + /// Writer-owned logical bytes of the published package identity union. + #[doc(hidden)] + #[serde(skip)] + pub allocation_logical_bytes: u64, + /// Writer-owned distinct published package identities. + #[doc(hidden)] + #[serde(skip)] + pub allocation_physical_objects: u64, } /// Stable export result. @@ -421,6 +447,9 @@ impl GraphForge { }, selection_fingerprint: receipt.selection_fingerprint, output: request.output_path.clone(), + allocation_identity_allocated_bytes: receipt.allocation_identity_allocated_bytes, + allocation_logical_bytes: receipt.allocation_logical_bytes, + allocation_physical_objects: receipt.allocation_physical_objects, }) } @@ -526,6 +555,14 @@ impl GraphForge { transport_digest: receipt.transport_digest, generation_uuid: receipt.publication.generation_uuid, idempotent_replay: receipt.publication.idempotent_replay, + materialized_identity_allocated_bytes: receipt.materialized_identity_allocated_bytes, + published_identity_allocated_bytes: receipt.published_identity_allocated_bytes, + materialized_cleanup_removed_identity_allocated_bytes: receipt + .materialized_cleanup + .removed_identity_allocated_bytes, + materialized_cleanup_parent_sync_confirmed: receipt + .materialized_cleanup + .parent_sync_confirmed, }) } } diff --git a/crates/graphforge-api/src/resumable_construction.rs b/crates/graphforge-api/src/resumable_construction.rs index 206d1db42..e14860146 100644 --- a/crates/graphforge-api/src/resumable_construction.rs +++ b/crates/graphforge-api/src/resumable_construction.rs @@ -270,11 +270,7 @@ impl GraphConstructionSession<'_> { } let (prepared_dir, prepared_guard, hydration_evidence) = super::hydrate_graph_workspace(&resolved, false)?; - self.inner.record_hydration_application_read_bytes( - hydration_evidence - .bytes_validated - .saturating_add(hydration_evidence.bytes_copied), - )?; + self.inner.record_hydration_evidence(&hydration_evidence)?; let runtime_catalog = super::load_runtime_catalog(&prepared_dir)?; let property_inventory = std::sync::Arc::new( graphforge_storage::AuthenticatedPropertyInventory::from_resolved_generation( diff --git a/crates/graphforge-api/tests/scale_g500_ladder.rs b/crates/graphforge-api/tests/scale_g500_ladder.rs index 2ce37a691..193843858 100644 --- a/crates/graphforge-api/tests/scale_g500_ladder.rs +++ b/crates/graphforge-api/tests/scale_g500_ladder.rs @@ -16,8 +16,10 @@ //! certify one billion live edges — that is #745. Small rungs run in normal CI; //! large rungs are opt-in via `make bench-g500-ladder`. +#![recursion_limit = "256"] + use std::cmp::Reverse; -use std::collections::BinaryHeap; +use std::collections::{BTreeMap, BinaryHeap}; use std::fs::{self, File}; use std::io::{BufReader, BufWriter, ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; @@ -29,6 +31,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use arrow::array::{Array, FixedSizeBinaryArray, Int64Array, StringArray, UInt64Array}; use arrow::record_batch::RecordBatch; + use graphforge_api::{ CONSTRUCTION_EDGE_SCHEMA, CONSTRUCTION_NODE_SCHEMA, CancellationToken, GraphConstructionBudgets, GraphConstructionSession, GraphForge, OperationId, PortableSelection, @@ -497,6 +500,179 @@ struct RungOutcome { evidence: Value, } +fn exact_descriptor_allocation(paths: &[PathBuf]) -> Value { + let mut logical_bytes = 0_u64; + let mut allocated = 0_u64; + for path in paths { + logical_bytes = logical_bytes.saturating_add( + fs::metadata(path) + .expect("generator descriptor metadata") + .len(), + ); + let file = File::open(path).expect("open exact allocation descriptor"); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + allocated = allocated.saturating_add( + file.metadata() + .expect("exact descriptor metadata") + .blocks() + .saturating_mul(512), + ); + } + #[cfg(not(unix))] + panic!("certification descriptor allocation requires Unix stat blocks"); + } + json!({ + "category": "generator_spill", + "logical_bytes": logical_bytes, + "allocated_bytes": allocated, + "logical_references": paths.len(), + "physical_objects": paths.len(), + "source": "generator_exact_descriptors", + }) +} + +fn exact_descriptor_identities(paths: &[PathBuf]) -> BTreeMap { + paths + .iter() + .map(|path| { + let file = File::open(path).expect("open exact allocation descriptor"); + let identity = graphforge_filesystem::file_identity(&file) + .expect("exact descriptor native identity"); + let allocation = graphforge_filesystem::file_space_usage(&file) + .expect("exact descriptor allocation") + .allocated_bytes; + let mut file_id = String::with_capacity(32); + for byte in identity.file_id { + use std::fmt::Write as _; + write!(&mut file_id, "{byte:02x}").expect("write identity string"); + } + ( + format!("{:016x}:{file_id}", identity.volume_serial), + allocation, + ) + }) + .collect() +} + +fn portable_export_allocation(receipt: &graphforge_api::PortableV2ExportFacadeResult) -> Value { + json!({ + "category": "portable_package", + "logical_bytes": receipt.allocation_logical_bytes, + "allocated_bytes": receipt.allocation_identity_allocated_bytes.values().copied().sum::(), + "logical_references": receipt.allocation_physical_objects, + "physical_objects": receipt.allocation_physical_objects, + "source": "portable_writer_receipt", + }) +} + +fn storage_attribution_value(project: &Path) -> Value { + let mut value = + serde_json::to_value(storage_attribution(project)).expect("serialize storage attribution"); + value + .as_object_mut() + .expect("storage attribution object") + .remove("generation_uuid"); + value + .as_object_mut() + .expect("storage attribution object") + .remove("physical_identity_allocated_bytes"); + value +} + +fn reject_unsanitized_evidence(value: &Value) -> Result<(), String> { + fn visit(value: &Value, trail: &str) -> Result<(), String> { + match value { + Value::Object(fields) => { + for (key, child) in fields { + let normalized = key.to_ascii_lowercase(); + if [ + "secret", + "credential", + "password", + "token", + "machine_id", + "volume_id", + "provider_resource_id", + "absolute_path", + "host_path", + ] + .iter() + .any(|needle| normalized.contains(needle)) + { + return Err(format!("sensitive evidence key at {trail}.{key}")); + } + visit(child, &format!("{trail}.{key}"))?; + } + } + Value::Array(items) => { + for (index, child) in items.iter().enumerate() { + visit(child, &format!("{trail}[{index}]"))?; + } + } + Value::String(text) => { + if Uuid::parse_str(text).is_ok() { + return Err(format!("raw UUID at {trail}")); + } + if text.starts_with('/') + || text.starts_with("\\\\") + || (text.len() >= 3 + && text.as_bytes()[1] == b':' + && matches!(text.as_bytes()[2], b'/' | b'\\')) + || text.split_whitespace().any(|part| part.starts_with('/')) + { + return Err(format!("absolute host path at {trail}")); + } + } + _ => {} + } + Ok(()) + } + visit(value, "$") +} + +fn sanitized_construction_evidence( + evidence: &graphforge_storage::GraphConstructionEvidence, +) -> Value { + let mut value = serde_json::to_value(evidence).expect("serialize construction evidence"); + value + .as_object_mut() + .expect("construction evidence object") + .remove("storage_active_identity_allocated_bytes"); + value + .as_object_mut() + .expect("construction evidence object") + .remove("storage_allocation_transitions"); + value +} + +fn storage_attribution(project: &Path) -> graphforge_storage::StorageAttributionSnapshot { + let graph = GraphForge::new(project.to_str()).expect("open attribution facade"); + let snapshot = graph + .storage_attribution() + .expect("capture authenticated storage attribution through public facade"); + snapshot + .validate_reconciliation() + .expect("storage attribution reconciliation"); + assert!( + snapshot.is_fully_classified(), + "qualification refuses unclassified retained artifacts" + ); + snapshot +} + +#[test] +fn public_storage_attribution_is_generation_bound_and_fully_classified() { + let project = TempDir::new().expect("storage attribution project"); + let graph = GraphForge::new(project.path().to_str()).expect("open attribution facade"); + let snapshot = graph + .storage_attribution() + .expect("public facade storage attribution"); + snapshot.validate_for_qualification().unwrap(); + assert!(snapshot.is_fully_classified()); +} + /// Check the envelope after a phase. Returns `Some(error_class)` on the first /// violation so the caller can stop the ladder. `ladder_started` is the /// ladder-level clock so the 4 h wall-clock fail-safe bounds the whole run, not each @@ -504,14 +680,12 @@ struct RungOutcome { fn envelope_violation( env: &RunEnvelope, ladder_started: Instant, - project: &Path, - spill: &Path, + disk_used_bytes: u64, ) -> Option<&'static str> { if peak_rss().is_some_and(|(rss, _)| rss > env.rss_bytes) { return Some("oom"); } - let disk = directory_bytes(project).unwrap_or(0) + directory_bytes(spill).unwrap_or(0); - if disk > env.disk_bytes { + if disk_used_bytes > env.disk_bytes { return Some("disk_exhaustion"); } if ladder_started.elapsed().as_secs() > env.timeout_s { @@ -681,8 +855,6 @@ impl IngestHeartbeat { rung: &Rung, completed_rungs: &[Value], steps: &[Value], - project: &Path, - spill: &Path, ) -> Self { let Ok(path) = std::env::var("GF_G500_LADDER_JOURNAL_OUT") else { return Self { @@ -698,8 +870,6 @@ impl IngestHeartbeat { let scale = rung.scale; let completed_rungs = completed_rungs.to_vec(); let steps = steps.to_vec(); - let project = project.to_path_buf(); - let spill = spill.to_path_buf(); let handle = thread::spawn(move || { loop { let value = json!({ @@ -714,8 +884,6 @@ impl IngestHeartbeat { "active_chunk_index": INGEST_CHUNK_INDEX.load(Ordering::Relaxed), "process_memory": linux_process_memory(), "storage_io": storage_io_value(), - "disk_used_bytes": directory_bytes(&project).unwrap_or(0) - .saturating_add(directory_bytes(&spill).unwrap_or(0)), "completed_rungs": completed_rungs, "active_steps": steps, "first_failing_phase": null, @@ -815,7 +983,11 @@ fn run_rung( None, ); let generate_s = gen_started.elapsed().as_secs_f64(); - let gen_violation = envelope_violation(&env, ladder_started, &project, &spill_dir); + let generator_allocation = exact_descriptor_allocation(&spill.runs); + let generator_allocated_bytes = generator_allocation["allocated_bytes"] + .as_u64() + .expect("generator allocated bytes"); + let gen_violation = envelope_violation(&env, ladder_started, generator_allocated_bytes); if let Some(class) = gen_violation { first_failing_phase = Some("generate"); error_class = Some(class); @@ -831,6 +1003,7 @@ fn run_rung( "peak_buffer_len": spill.peak_buffer_len, "buffer_edges": rung.buffer_edges, "run_count": spill.runs.len(), + "storage": generator_allocation, } })); persist_phase_journal( @@ -868,8 +1041,7 @@ fn run_rung( graphforge_storage::io_stats::reset(); INGEST_CHUNK_INDEX.store(0, Ordering::Relaxed); INGEST_SUBPHASE.store(1, Ordering::Relaxed); - let heartbeat = - IngestHeartbeat::start(profile, rung, completed_rungs, &steps, &project, &spill_dir); + let heartbeat = IngestHeartbeat::start(profile, rung, completed_rungs, &steps); let mut construction = open_persisted_construction( &graph, &spill_dir.join("construction-session.uuid"), @@ -909,9 +1081,21 @@ fn run_rung( INGEST_SUBPHASE.store(0, Ordering::Relaxed); heartbeat.stop(); drop(graph); + let committed_snapshot = storage_attribution(&project); + let ingest_disk_used_bytes = + generator_allocated_bytes.saturating_add(committed_snapshot.allocated_bytes); + let committed_storage = + serde_json::to_value(committed_snapshot).expect("serialize committed storage"); + let construction_phases = + graphforge_storage::ConstructionPhaseAttribution::from_construction( + &construction_evidence, + ); + construction_phases + .validate_reconciliation() + .expect("construction phase attribution reconciliation"); ingest_ran = true; let ingest_s = ingest_started.elapsed().as_secs_f64(); - let ingest_violation = envelope_violation(&env, ladder_started, &project, &spill_dir); + let ingest_violation = envelope_violation(&env, ladder_started, ingest_disk_used_bytes); if let Some(class) = ingest_violation { first_failing_phase = Some("ingest"); error_class = Some(class); @@ -921,8 +1105,7 @@ fn run_rung( "pass": ingest_violation.is_none(), "wall_time_s": ingest_s, "rss_peak_bytes": rss_value(), - "disk_used_bytes": directory_bytes(&project).unwrap_or(0) - .saturating_add(directory_bytes(&spill_dir).unwrap_or(0)), + "disk_used_bytes": ingest_disk_used_bytes, "detail": { "live_unique_edges": live_unique_edges, "duplicates_rejected": duplicates_rejected, @@ -959,7 +1142,10 @@ fn run_rung( "parquet_write_operations": construction_evidence.parquet_write_operations, "retained_probe_read_bytes": construction_evidence.retained_probe_read_bytes, "retained_probe_block_loads": construction_evidence.retained_probe_block_loads, + "storage_transient_peak_allocated_bytes": construction_evidence.storage_transient_peak_allocated_bytes, }, + "committed_storage": committed_storage, + "application_io_phases": construction_phases, } })); persist_phase_journal( @@ -998,7 +1184,9 @@ fn run_rung( edge_count = scalar_count(&graph.execute(COUNT_EDGES).expect("edge count")); let reopen_s = reopen_started.elapsed().as_secs_f64(); gsi = gsi_undirected(node_count, edge_count); - let reopen_violation = envelope_violation(&env, ladder_started, &project, &spill_dir); + let reopen_disk_used_bytes = + generator_allocated_bytes.saturating_add(storage_attribution(&project).allocated_bytes); + let reopen_violation = envelope_violation(&env, ladder_started, reopen_disk_used_bytes); if let Some(class) = reopen_violation { first_failing_phase = Some("reopen"); error_class = Some(class); @@ -1054,7 +1242,9 @@ fn run_rung( "wall_time_s": hop2_started.elapsed().as_secs_f64(), "detail": { "rows": hop2_rows } })); - let query_violation = envelope_violation(&env, ladder_started, &project, &spill_dir); + let query_disk_used_bytes = generator_allocated_bytes + .saturating_add(storage_attribution(&project).allocated_bytes); + let query_violation = envelope_violation(&env, ladder_started, query_disk_used_bytes); if let Some(class) = query_violation { first_failing_phase = Some("query"); error_class = Some(class); @@ -1076,8 +1266,11 @@ fn run_rung( drop(graph); } - let disk_used_bytes = - directory_bytes(&project).unwrap_or(0) + directory_bytes(&spill_dir).unwrap_or(0); + let disk_used_bytes = generator_allocated_bytes.saturating_add( + ingest_ran + .then(|| storage_attribution(&project).allocated_bytes) + .unwrap_or(0), + ); // Tri-state: reconciliation is only *evaluated* once ingest has run. A rung // stopped in the generate phase is reported as null (not evaluated), never // as a forced `true`. @@ -1423,26 +1616,6 @@ fn git_sha() -> Value { .map_or(Value::Null, |sha| Value::String(sha.trim().to_owned())) } -fn directory_bytes(path: &Path) -> std::io::Result { - if !path.exists() { - return Ok(0); - } - if path.is_file() { - return Ok(path.metadata()?.len()); - } - let mut total = 0u64; - for entry in fs::read_dir(path)? { - let entry = entry?; - let metadata = entry.metadata()?; - total += if metadata.is_dir() { - directory_bytes(&entry.path())? - } else { - metadata.len() - }; - } - Ok(total) -} - /// Returns `(bytes, source)`. `"vmhwm"` (Linux `/proc/self/status`) is a true /// high-water mark; `"ps_sampled"` (fallback) is the instantaneous RSS at the /// moment of the call, i.e. a **lower bound** on the real peak. Consumers must @@ -1894,20 +2067,25 @@ struct PhaseJournal { path: PathBuf, phases: Vec, monitor: ResourceMonitor, + allocation: graphforge_storage::StorageAllocationLifecycle, } impl PhaseJournal { - fn new(path: PathBuf, workspace: &Path, envelope: Envelope) -> Self { + fn new(path: PathBuf, _workspace: &Path, envelope: Envelope) -> Self { Self { path, phases: Vec::new(), - monitor: ResourceMonitor::start(workspace.to_path_buf(), envelope), + monitor: ResourceMonitor::start(envelope), + allocation: graphforge_storage::StorageAllocationLifecycle::default(), } } fn pass(&mut self, id: &str, started: Instant, fingerprint: Option) { let fingerprint = fingerprint.map_or(Value::Null, Value::String); - self.monitor.sample_disk(); + // Every phase owns the live allocation union for its full duration, + // even when it does not install or remove an allocation identity. + self.monitor + .observe_allocated_union(self.allocation.current_allocated_bytes()); if let Some(code) = self.monitor.failure_code() { self.phases.push(json!({ "id": id, "status": "fail", @@ -1940,6 +2118,50 @@ impl PhaseJournal { self.monitor.cancellation.clone() } + fn replace_allocation_owner(&mut self, owner: &str, identities: &BTreeMap) { + self.allocation + .replace_owner(owner, identities) + .expect("replace exact allocation owner"); + self.monitor + .observe_allocated_union(self.allocation.current_allocated_bytes()); + } + + fn replace_project_owner( + &mut self, + owner: &str, + generation: &graphforge_storage::ResolvedProjectGeneration, + ) { + let project = graphforge_storage::capture_project_storage_identity_union(generation) + .expect("capture retained project identity union"); + self.replace_allocation_owner(owner, &project.physical_identity_allocated_bytes); + } + + fn replay_allocation_transitions( + &mut self, + owner: &str, + transitions: &[graphforge_storage::StorageAllocationTransition], + ) { + for transition in transitions { + self.allocation + .apply_owner_transition(owner, transition) + .expect("apply writer-owned allocation transition"); + self.monitor + .observe_allocated_union(self.allocation.current_allocated_bytes()); + } + } + + fn remove_allocation_owner(&mut self, owner: &str) { + self.allocation + .remove_owner(owner) + .expect("remove exact allocation owner"); + self.monitor + .observe_allocated_union(self.allocation.current_allocated_bytes()); + } + + fn current_allocated_union(&self) -> u64 { + self.allocation.current_allocated_bytes() + } + fn flush(&self) { let staged = self.path.with_extension("json.tmp"); fs::write( @@ -1965,7 +2187,6 @@ impl Drop for PhaseJournal { .failure_code() .or_else(|| std::thread::panicking().then_some("operation_failed")); if let Some(code) = failure_code { - self.monitor.sample_disk(); self.phases.push(json!({ "id": CERTIFICATION_PHASES[self.phases.len()], "status": "fail", "elapsed_ms": 0, @@ -1979,7 +2200,6 @@ impl Drop for PhaseJournal { } struct ResourceMonitor { - workspace: PathBuf, cancellation: CancellationToken, stop: Arc, peak_rss: Arc, @@ -1990,29 +2210,24 @@ struct ResourceMonitor { } impl ResourceMonitor { - fn start(workspace: PathBuf, envelope: Envelope) -> Self { + fn start(envelope: Envelope) -> Self { let initial_rss = current_rss_bytes().expect("certification host must expose process RSS"); - let initial_disk = allocated_bytes(&workspace) - .expect("certification host must expose allocated disk bytes"); let cancellation = CancellationToken::new(); let stop = Arc::new(AtomicBool::new(false)); let peak_rss = Arc::new(AtomicU64::new(initial_rss)); - let peak_disk = Arc::new(AtomicU64::new(initial_disk)); + let peak_disk = Arc::new(AtomicU64::new(0)); let failure = Arc::new(AtomicU64::new(0)); let worker_cancellation = cancellation.clone(); let worker_stop = Arc::clone(&stop); let worker_peak_rss = Arc::clone(&peak_rss); - let worker_peak_disk = Arc::clone(&peak_disk); let worker_failure = Arc::clone(&failure); - let worker_workspace = workspace.clone(); let started = Instant::now(); let elapsed_before_process = certification_elapsed_before_process(); let worker = thread::spawn(move || { - let mut samples = 0_u8; while !worker_stop.load(Ordering::Relaxed) { let rss = current_rss_bytes().expect("certification RSS probe failed"); worker_peak_rss.fetch_max(rss, Ordering::Relaxed); - let mut code = if rss > envelope.rss_bytes { + let code = if rss > envelope.rss_bytes { 1 } else if elapsed_before_process .saturating_add(started.elapsed()) @@ -2023,14 +2238,6 @@ impl ResourceMonitor { } else { 0 }; - if samples == 0 { - let disk = allocated_bytes(&worker_workspace) - .expect("certification disk probe failed"); - worker_peak_disk.fetch_max(disk, Ordering::Relaxed); - if disk > envelope.disk_bytes { - code = 2; - } - } if code != 0 { worker_failure .compare_exchange(0, code, Ordering::SeqCst, Ordering::Relaxed) @@ -2038,12 +2245,10 @@ impl ResourceMonitor { worker_cancellation.cancel(); break; } - samples = (samples + 1) % 20; thread::sleep(Duration::from_millis(250)); } }); Self { - workspace, cancellation, stop, peak_rss, @@ -2054,10 +2259,9 @@ impl ResourceMonitor { } } - fn sample_disk(&self) { - let disk = allocated_bytes(&self.workspace).expect("certification disk probe failed"); - self.peak_disk.fetch_max(disk, Ordering::Relaxed); - if disk > self.envelope.disk_bytes { + fn observe_allocated_union(&self, bytes: u64) { + self.peak_disk.fetch_max(bytes, Ordering::Relaxed); + if bytes > self.envelope.disk_bytes { self.failure .compare_exchange(0, 2, Ordering::SeqCst, Ordering::Relaxed) .ok(); @@ -2114,23 +2318,6 @@ impl Drop for ResourceMonitor { } } -fn allocated_bytes(path: &Path) -> Result { - let output = Command::new("du").arg("-sk").arg(path).output(); - output - .ok() - .filter(|out| out.status.success()) - .and_then(|out| { - String::from_utf8(out.stdout) - .ok()? - .split_whitespace() - .next()? - .parse::() - .ok() - }) - .map(|kibibytes| kibibytes.saturating_mul(1024)) - .ok_or("allocated disk usage is unavailable") -} - fn result_fingerprint(result: &graphforge_api::ExecutionResult) -> String { let mut hasher = Sha256::new(); if let Some(batch) = result.batches.first() { @@ -2207,7 +2394,17 @@ fn current_generation_uuid(graph: &GraphForge) -> Uuid { Uuid::from_bytes(bytes) } -fn create_bounded_drill_package(root: &Path, limits: PortableV2Limits) -> (PathBuf, String) { +struct DrillAllocationEvidence { + project: BTreeMap, + construction: BTreeMap, + expanded: BTreeMap, + cancelled_export: BTreeMap, +} + +fn create_bounded_drill_package( + root: &Path, + limits: PortableV2Limits, +) -> (PathBuf, String, DrillAllocationEvidence) { let project = root.join("drill-source"); let package = root.join("drill.gfpb"); fs::create_dir_all(&project).expect("bounded drill project"); @@ -2225,11 +2422,21 @@ fn create_bounded_drill_package(root: &Path, limits: PortableV2Limits) -> (PathB construction .seal_and_publish() .expect("publish bounded drill construction"); + let construction_identities = construction + .progress() + .evidence + .storage_active_identity_allocated_bytes; drop(construction); drop(graph); let graph = GraphForge::new(project.to_str()).expect("reopen bounded drill project"); + let project_generation = graphforge_storage::resolve_project_generation(&project) + .expect("resolve bounded drill project"); + let project_identities = + graphforge_storage::capture_project_storage_identity_union(&project_generation) + .expect("bounded drill retained project attribution") + .physical_identity_allocated_bytes; let expanded = root.join("drill-expanded"); - graph + let expanded_receipt = graph .export_portable_v2( &PortableV2ExportRequest { selection: PortableSelection::Current, @@ -2245,31 +2452,31 @@ fn create_bounded_drill_package(root: &Path, limits: PortableV2Limits) -> (PathB .expect("export compact drill expanded package"); verify_portable_v2( &PortableVerifyRequest { - input: expanded, + input: expanded.clone(), mode: PortableV2Mode::Full, limits, }, None, ) .expect("verify compact drill expanded package"); + let expanded_identities = expanded_receipt.allocation_identity_allocated_bytes; + fs::remove_dir_all(&expanded).expect("remove bounded expanded drill package"); let cancelled = AtomicBool::new(true); let cancelled_path = root.join("drill-cancelled.gfpb"); - assert!( - graph - .export_portable_v2( - &PortableV2ExportRequest { - selection: PortableSelection::Current, - output_path: cancelled_path.clone(), - representation: PortableV2Output::Bundle, - profile: PortableV2SelectionProfile::Complete, - subset: None, - limits, - }, - Some(&cancelled), - |_| {}, - ) - .is_err() - ); + let cancelled_error = graph + .export_portable_v2( + &PortableV2ExportRequest { + selection: PortableSelection::Current, + output_path: cancelled_path.clone(), + representation: PortableV2Output::Bundle, + profile: PortableV2SelectionProfile::Complete, + subset: None, + limits, + }, + Some(&cancelled), + |_| {}, + ) + .expect_err("cancelled drill export must fail"); assert!(!cancelled_path.exists()); let receipt = graph .export_portable_v2( @@ -2285,11 +2492,29 @@ fn create_bounded_drill_package(root: &Path, limits: PortableV2Limits) -> (PathB |_| {}, ) .expect("export bounded drill package"); - (package, receipt.package_digest) + ( + package, + receipt.package_digest, + DrillAllocationEvidence { + project: project_identities, + construction: construction_identities, + expanded: expanded_identities, + cancelled_export: cancelled_error.allocation_identity_allocated_bytes, + }, + ) } #[allow(clippy::too_many_lines)] fn run_integrated_certification(root: &Path, target_live: Option) -> Value { + run_integrated_certification_with_edge_factor(root, target_live, None) +} + +#[allow(clippy::too_many_lines)] +fn run_integrated_certification_with_edge_factor( + root: &Path, + target_live: Option, + preflight_edge_factor: Option, +) -> Value { let source = root.join("source"); let imported = root.join("imported"); let package = root.join("project.gfpb"); @@ -2321,7 +2546,7 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value let edge_factor = if target_live.is_some() { certification_profile.edgefactor } else { - 4 + preflight_edge_factor.unwrap_or(4) }; let initiator = if target_live.is_some() { certification_profile.initiator @@ -2364,10 +2589,19 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value || target_live_fingerprint.expect("target-live payload fingerprint"), |value| value.input_fingerprint.clone(), ); + if let Some(spills) = &spills { + journal.replace_allocation_owner( + "generator_spill", + &exact_descriptor_identities(&spills.runs), + ); + } journal.pass("generate", phase, Some(generation_fingerprint.clone())); let phase = Instant::now(); let graph = GraphForge::new(source.to_str()).expect("open certification source"); + let initial_generation = graphforge_storage::resolve_project_generation(&source) + .expect("resolve initial source generation"); + journal.replace_project_owner("source_project", &initial_generation); let mut construction = graph .begin_graph_construction(Default::default()) .expect("begin certification construction"); @@ -2395,12 +2629,36 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value construction .seal_and_publish() .expect("publish certification construction"); + let construction_evidence = construction.progress().evidence; + let mut construction_phases = + graphforge_storage::ConstructionPhaseAttribution::from_construction(&construction_evidence); + construction_phases + .validate_for_qualification() + .expect("certification construction phase attribution"); + let pre_construction_union = journal.current_allocated_union(); + journal.replay_allocation_transitions( + "construction", + &construction_evidence.storage_allocation_transitions, + ); + // Construction artifacts are private to this session and cannot alias the + // already-open source project. The storage-owned numeric high-water mark + // therefore restores peaks compacted out of durable checkpoint history. + journal.monitor.observe_allocated_union( + pre_construction_union + .saturating_add(construction_evidence.storage_transient_peak_total_allocated_bytes), + ); + let committed_generation = graphforge_storage::resolve_project_generation(&source) + .expect("resolve committed ingest generation"); + journal.replace_project_owner("source_project", &committed_generation); journal.pass("ingest", phase, Some(input_fingerprint)); let phase = Instant::now(); let csr = graph .rebuild_adjacency(Some(journal.cancellation_token())) .expect("build certification CSR"); + let csr_generation = graphforge_storage::resolve_project_generation(&source) + .expect("resolve committed CSR generation"); + journal.replace_project_owner("source_project", &csr_generation); journal.pass( "csr", phase, @@ -2449,6 +2707,10 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value ) .expect("portable-v2 export"); assert_eq!(source_generation, exported.generation_uuid); + journal.replace_allocation_owner( + "portable_package", + &exported.allocation_identity_allocated_bytes, + ); journal.pass("export", phase, Some(exported.package_digest.clone())); let phase = Instant::now(); let verified = verify_portable_v2( @@ -2475,6 +2737,23 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value Some(journal.cancellation()), ) .expect("atomic portable-v2 import"); + // Replay the storage-owned operation transitions in their actual order: + // private materialization coexisted with the published generation until + // deterministic staging cleanup completed. + journal.replace_allocation_owner( + "import_materialized", + &imported_receipt.materialized_identity_allocated_bytes, + ); + journal.replace_allocation_owner( + "clean_import", + &imported_receipt.published_identity_allocated_bytes, + ); + assert!(imported_receipt.materialized_cleanup_parent_sync_confirmed); + assert_eq!( + imported_receipt.materialized_cleanup_removed_identity_allocated_bytes, + imported_receipt.materialized_identity_allocated_bytes + ); + journal.remove_allocation_owner("import_materialized"); assert_ne!(exported.generation_uuid, imported_receipt.generation_uuid); journal.pass( "import", @@ -2483,6 +2762,9 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value ); let phase = Instant::now(); let imported_graph = GraphForge::new(imported.to_str()).expect("reopen import"); + let imported_generation = graphforge_storage::resolve_project_generation(&imported) + .expect("resolve clean import generation"); + journal.replace_project_owner("clean_import_project", &imported_generation); let imported_nodes = imported_graph .node_count(NODE_LABEL) .expect("imported nodes"); @@ -2507,11 +2789,31 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value assert_eq!(source_2hop, imported_2hop); assert_eq!(source_authority_fingerprint, imported_authority_fingerprint); journal.pass("imported_query_2hop", phase, Some(imported_2hop.clone())); - + let source_storage = storage_attribution_value(&source); + let source_project_current_allocated_bytes = + graphforge_storage::capture_project_storage_identity_union( + &graphforge_storage::resolve_project_generation(&source) + .expect("resolve authoritative source project"), + ) + .expect("capture authoritative source project identity union") + .allocated_bytes; + let imported_storage = storage_attribution_value(&imported); + let package_storage = portable_export_allocation(&exported); // Representative drills use the same verifier/import boundaries but never // repeat the billion-edge payload. let phase = Instant::now(); - let (drill_package, drill_digest) = create_bounded_drill_package(root, limits); + let (drill_package, drill_digest, drill_allocation) = + create_bounded_drill_package(root, limits); + journal.replace_allocation_owner("drill_project", &drill_allocation.project); + journal.replace_allocation_owner("drill_construction", &drill_allocation.construction); + journal.replace_allocation_owner("drill_expanded", &drill_allocation.expanded); + journal.remove_allocation_owner("drill_expanded"); + journal.replace_allocation_owner("drill_cancelled_export", &drill_allocation.cancelled_export); + journal.remove_allocation_owner("drill_cancelled_export"); + journal.replace_allocation_owner( + "drill_package", + &exact_descriptor_identities(std::slice::from_ref(&drill_package)), + ); let drill_verified = verify_portable_v2( &PortableVerifyRequest { input: drill_package.clone(), @@ -2532,6 +2834,10 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value file.write_all(b"corruption").expect("append corruption"); file.flush().expect("flush corruption"); } + journal.replace_allocation_owner( + "corrupt_drill_package", + &exact_descriptor_identities(std::slice::from_ref(&corrupt)), + ); assert!( verify_portable_v2( &PortableVerifyRequest { @@ -2577,18 +2883,55 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value journal.pass("drill_resource_limit", phase, None); let phase = Instant::now(); let interrupted = root.join("interrupted-target"); - assert!( - GraphForge::import_portable_v2( - &interrupted, - &PortableV2ImportRequest { - input: drill_package, - operation_id: OperationId(uuidv7(0x746)), - limits, - }, - Some(&AtomicBool::new(true)) - ) - .is_err() + let interrupted_operation = uuidv7(0x746); + let interrupted_generation = Uuid::new_v5( + &interrupted_operation, + b"graphforge-portable-v2-import-generation/1", + ); + let interrupted_cancelled = AtomicBool::new(false); + let supported_capabilities = [ + "epistemic", + "graph", + "knowledge", + "provenance", + "valid_time", + "workspace", + ] + .into_iter() + .map(|capability_id| graphforge_storage::ProjectCapability { + capability_id: capability_id.into(), + capability_version: 1, + }) + .collect::>(); + let interrupted_error = graphforge_storage::import_complete_portable_v2_with_progress( + &drill_package, + &interrupted, + interrupted_operation, + interrupted_generation, + &supported_capabilities, + limits, + Some(&interrupted_cancelled), + |progress| { + if progress.phase == graphforge_storage::PortableV2ImportPhase::Materialized { + interrupted_cancelled.store(true, Ordering::SeqCst); + } + }, + ) + .expect_err("cancelled finalization must fail"); + assert!(interrupted_error.recovery_reauthentication_read_bytes > 0); + assert!(interrupted_error.recovery_reauthentication_read_calls > 0); + construction_phases.add_recovery_reauthentication( + interrupted_error.recovery_reauthentication_read_bytes, + interrupted_error.recovery_reauthentication_read_calls, + ); + construction_phases + .validate_for_qualification() + .expect("interrupted recovery phase attribution"); + journal.replace_allocation_owner( + "interrupted_import", + &interrupted_error.allocation_identity_allocated_bytes, ); + journal.remove_allocation_owner("interrupted_import"); assert!(!interrupted.join("CURRENT").exists()); journal.pass("drill_interrupted_finalization", phase, None); @@ -2601,10 +2944,12 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value digest.update(two_hop.as_bytes()); format!("sha256:{}", hex_encode(digest.finalize())) }; - json!({ - "source_generation": exported.generation_uuid.to_string(), + let workspace_current_allocated_bytes = journal.current_allocated_union(); + let evidence = json!({ + "source_export_generation_authenticated": source_generation == exported.generation_uuid, + "import_receipt_reopen_authenticated": current_generation_uuid(&imported_graph) == imported_receipt.generation_uuid, + "source_import_generations_distinct": exported.generation_uuid != imported_receipt.generation_uuid, "package": exported.package_digest, "transport": exported.transport_digest, - "imported_generation": imported_receipt.generation_uuid.to_string(), "raw_attempts": spills.as_ref().map_or_else(|| summary.as_ref().unwrap().raw_attempts, |value| value.raw_attempts), "self_loops_rejected": spills.as_ref().map_or_else(|| summary.as_ref().unwrap().self_loops_rejected, |value| value.self_loops_rejected), "duplicates_rejected": generated_counts.as_ref().map_or_else(|| summary.as_ref().unwrap().duplicates_rejected, |value| value.duplicates_rejected), @@ -2619,8 +2964,19 @@ fn run_integrated_certification(root: &Path, target_live: Option) -> Value "compatibility": serde_json::to_value(verified.compatibility).expect("compatibility JSON"), "source_authority_fingerprint": source_authority_fingerprint, "imported_authority_fingerprint": imported_authority_fingerprint, + "storage": { + "source": source_storage, + "source_project_current_allocated_bytes": source_project_current_allocated_bytes, + "portable_package": package_storage, + "clean_import": imported_storage, + "construction": sanitized_construction_evidence(&construction_evidence), + "application_io_phases": construction_phases, + "workspace_current_allocated_bytes": workspace_current_allocated_bytes, + }, "phases": journal.phases, - }) + }); + reject_unsanitized_evidence(&evidence).expect("certification lifecycle evidence is sanitized"); + evidence } #[test] @@ -2628,16 +2984,128 @@ fn certification_lifecycle_journals_equivalent_round_trip_and_drills() { let root = TempDir::new().expect("certification smoke root"); let evidence = run_integrated_certification(root.path(), None); assert_eq!(evidence["source_edges"], evidence["imported_edges"]); - assert_ne!( - evidence["source_generation"], - evidence["imported_generation"] - ); + assert_eq!(evidence["source_export_generation_authenticated"], true); + assert_eq!(evidence["import_receipt_reopen_authenticated"], true); + assert_eq!(evidence["source_import_generations_distinct"], true); + reject_unsanitized_evidence(&evidence).expect("lifecycle evidence remains sanitized"); +} + +#[test] +fn certification_evidence_sanitizer_rejects_identity_paths_and_sensitive_keys() { + for (value, expected) in [ + ( + json!({"proof": "018f6e45-7f12-7c00-8000-000000000001"}), + "raw UUID", + ), + ( + json!({"proof": "/var/lib/graphforge/project"}), + "absolute host path", + ), + ( + json!({"nested": {"api_token": "redacted"}}), + "sensitive evidence key", + ), + ] { + let error = reject_unsanitized_evidence(&value).expect_err("unsafe evidence must fail"); + assert!( + error.contains(expected), + "unexpected sanitizer failure: {error}" + ); + } + reject_unsanitized_evidence(&json!({ + "generation_authenticated": true, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + })) + .expect("closed proof is safe"); +} + +#[test] +fn equivalent_full_lifecycle_1x_2x_4x_has_bounded_phase_slopes() { + const FIELDS: [&str; 7] = [ + "read_bytes", + "write_bytes", + "read_calls", + "write_calls", + "object_count", + "block_count", + "fsync_calls", + ]; + let mut baseline: Option> = None; + for factor in [1_u32, 2, 4] { + let root = TempDir::new().expect("full lifecycle ladder root"); + let evidence = + run_integrated_certification_with_edge_factor(root.path(), None, Some(factor)); + assert_eq!(evidence["source_edges"], evidence["imported_edges"]); + let phases = evidence["storage"]["application_io_phases"]["phases"] + .as_object() + .expect("phase evidence object"); + let attribution: graphforge_storage::ConstructionPhaseAttribution = + serde_json::from_value(evidence["storage"]["application_io_phases"].clone()) + .expect("decode phase evidence"); + attribution + .validate_for_qualification() + .expect("full lifecycle phase qualification"); + let recovery = &phases["recovery_reauthentication"]; + assert!( + recovery["read_bytes"].as_u64().unwrap_or(0) > 0, + "{factor}x interrupted-finalization recovery must report authenticated bytes" + ); + assert!( + recovery["read_calls"].as_u64().unwrap_or(0) > 0, + "{factor}x interrupted-finalization recovery must report authenticated calls" + ); + let observations = phases + .iter() + .map(|(name, values)| { + let counters = std::array::from_fn(|index| { + values[FIELDS[index]] + .as_u64() + .expect("phase counter is an integer") + }); + (name.clone(), counters) + }) + .collect::>(); + if let Some(base) = &baseline { + assert_eq!( + base.keys().collect::>(), + observations.keys().collect::>() + ); + for (phase, current) in &observations { + for (index, value) in current.iter().enumerate() { + let first = base[phase][index]; + if first == 0 { + assert_eq!( + *value, 0, + "{phase}.{} appeared only at a larger rung", + FIELDS[index] + ); + } else { + assert!( + *value <= first.saturating_mul(u64::from(factor)).saturating_mul(2), + "{phase}.{} exceeded the documented 2x constant-factor ceiling", + FIELDS[index] + ); + } + } + } + } else { + baseline = Some(observations); + } + let interrupted = evidence["phases"] + .as_array() + .expect("lifecycle phases") + .iter() + .find(|phase| phase["id"] == "drill_interrupted_finalization") + .expect("interrupted-finalization recovery drill"); + assert_eq!(interrupted["status"], "pass"); + } } #[test] fn certification_watchdog_persists_typed_first_failure() { let root = TempDir::new().expect("watchdog root"); - fs::write(root.path().join("allocated.bin"), [0_u8; 4096]).expect("allocated fixture"); + let allocated = root.path().join("allocated.bin"); + fs::write(&allocated, [0_u8; 4096]).expect("allocated fixture"); let journal_path = root.path().join("journal.json"); let mut journal = PhaseJournal::new( journal_path.clone(), @@ -2648,6 +3116,10 @@ fn certification_watchdog_persists_typed_first_failure() { timeout_s: u64::MAX, }, ); + journal.replace_allocation_owner( + "watchdog_fixture", + &exact_descriptor_identities(&[allocated]), + ); let failure = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { journal.pass("resource_probe", Instant::now(), None); })); @@ -2802,6 +3274,32 @@ fn submitted_chunk_count(evidence: &graphforge_storage::GraphConstructionEvidenc .saturating_add(evidence.replayed_chunks) } +#[test] +fn active_ingest_heartbeat_does_not_recursively_scan_storage() { + let source = include_str!("scale_g500_ladder.rs"); + let heartbeat = source + .split("struct IngestHeartbeat") + .nth(1) + .and_then(|tail| tail.split("fn run_rung").next()) + .expect("heartbeat source boundary"); + let recursive_probe = ["directory", "bytes"].join("_"); + assert!( + !heartbeat.contains(&recursive_probe), + "active heartbeat must consume counters, not enumerate project paths" + ); + let monitor = source + .split("struct ResourceMonitor") + .nth(1) + .and_then(|tail| tail.split("fn certification_elapsed_before_process").next()) + .expect("resource monitor source boundary"); + for forbidden in ["read_dir", "walkdir", "du\""] { + assert!( + !monitor.contains(forbidden), + "resource monitor must use storage-owned observations, not {forbidden}" + ); + } +} + #[test] fn tiny_construction_ladder_resumes_and_scales_bounded_work_linearly() { let budgets = GraphConstructionBudgets { @@ -2812,6 +3310,8 @@ fn tiny_construction_ladder_resumes_and_scales_bounded_work_linearly() { }; let base_nodes = CONSTRUCTION_BATCH_ROWS as u64; let mut baseline_peaks: Option<[u64; 11]> = None; + let mut baseline_storage: Option<(u64, u64)> = None; + let mut baseline_phase_io: Option<(u64, u64, u64, u64)> = None; for factor in [1_u64, 2, 4] { let project = TempDir::new().expect("tiny construction project"); let graph = GraphForge::new(project.path().to_str()).expect("open tiny project"); @@ -2931,6 +3431,79 @@ fn tiny_construction_ladder_resumes_and_scales_bounded_work_linearly() { assert!(progress.evidence.parquet_write_operations > 0); assert_ne!(receipt.generation_uuid, before); assert_eq!(current_generation_uuid(&graph), receipt.generation_uuid); + let phases = + graphforge_storage::ConstructionPhaseAttribution::from_construction(&progress.evidence); + phases.validate_reconciliation().unwrap(); + let shape = + &phases.phases[&graphforge_storage::StorageIoPhase::ShapeConsumeReauthentication]; + assert!(progress.evidence.merge_read_operations > 0); + assert!(progress.evidence.merge_write_operations > 0); + assert_eq!( + shape.write_bytes, + progress + .evidence + .merge_written_bytes + .saturating_add(progress.evidence.parquet_write_bytes) + ); + assert_eq!( + shape.write_calls, + progress + .evidence + .merge_write_operations + .saturating_add(progress.evidence.parquet_write_operations) + ); + assert_eq!( + shape.read_calls, + progress + .evidence + .shape_input_validation_read_operations + .saturating_add(progress.evidence.merge_read_operations) + .saturating_add(progress.evidence.parquet_read_operations) + .saturating_add(progress.evidence.shaped_output_authentication_operations) + .saturating_add(progress.evidence.parent_catalog_read_operations) + .saturating_add(progress.evidence.retained_probe_block_loads) + ); + let phase_observation = ( + phases.totals.read_bytes, + phases.totals.write_bytes, + phases.totals.read_calls, + phases.totals.write_calls, + ); + if let Some(baseline) = baseline_phase_io { + // Each lifecycle has fixed authenticated control work. Preserve a + // documented 2x constant-factor ceiling around ideal linear growth + // instead of pretending the intercept is zero at the 1x fixture. + let ceiling = |base: u64| base.saturating_mul(factor).saturating_mul(2); + assert!(phase_observation.0 <= ceiling(baseline.0)); + assert!(phase_observation.1 <= ceiling(baseline.1)); + assert!(phase_observation.2 <= ceiling(baseline.2)); + assert!(phase_observation.3 <= ceiling(baseline.3)); + } else { + baseline_phase_io = Some(phase_observation); + } + let generation = graphforge_storage::resolve_project_generation(project.path()) + .expect("resolve tiny generation"); + let storage = graphforge_storage::capture_storage_attribution(&generation) + .expect("capture tiny storage attribution"); + storage + .validate_reconciliation() + .expect("reconcile tiny storage attribution"); + assert!( + storage.is_fully_classified(), + "unclassified tiny construction storage: {storage:#?}" + ); + if let Some((base_logical, base_allocated)) = baseline_storage { + assert!( + storage.logical_bytes <= base_logical.saturating_mul(factor), + "authenticated logical bytes exceeded linear growth" + ); + assert!( + storage.allocated_bytes <= base_allocated.saturating_mul(factor), + "deduplicated allocated bytes exceeded linear growth" + ); + } else { + baseline_storage = Some((storage.logical_bytes, storage.allocated_bytes)); + } drop(resumed); let replay = graph .resume_graph_construction(session_uuid, budgets) @@ -3056,7 +3629,12 @@ fn certification_target_live_full_lifecycle_evidence() { "source_nodes": lifecycle["source_nodes"], "source_edges": source_edges, "imported_nodes": lifecycle["imported_nodes"], "imported_edges": lifecycle["imported_edges"], }, - "identities": { "source_generation": lifecycle["source_generation"], "package": lifecycle["package"], "transport": lifecycle["transport"], "imported_generation": lifecycle["imported_generation"] }, + "identities": { + "source_export_generation_authenticated": lifecycle["source_export_generation_authenticated"], + "import_receipt_reopen_authenticated": lifecycle["import_receipt_reopen_authenticated"], + "source_import_generations_distinct": lifecycle["source_import_generations_distinct"], + "package": lifecycle["package"], "transport": lifecycle["transport"] + }, "package": { "contract": lifecycle["portable_contract"], "format": "portable-project-v2-bundle", "class": lifecycle["package_class"], "integrity": lifecycle["integrity"], @@ -3065,10 +3643,12 @@ fn certification_target_live_full_lifecycle_evidence() { }, "equivalence": { "source_project_fingerprint": lifecycle["source_project_fingerprint"], "imported_project_fingerprint": lifecycle["imported_project_fingerprint"] }, "authority": { "source_fingerprint": lifecycle["source_authority_fingerprint"], "imported_fingerprint": lifecycle["imported_authority_fingerprint"] }, + "storage_attribution": lifecycle["storage"], "phases": phases, - "envelope": { "peak_rss_bytes": peak_rss, "peak_disk_bytes": peak_disk, "wall_time_s": elapsed_before_process.saturating_add(started.elapsed()).as_secs_f64() }, + "envelope": { "peak_rss_bytes": peak_rss, "peak_disk_bytes": peak_disk, "peak_disk_source": "storage_owned_active_identity_union", "wall_time_s": elapsed_before_process.saturating_add(started.elapsed()).as_secs_f64() }, "result": "pass", "first_failure": null, }); + reject_unsanitized_evidence(&evidence).expect("provider certification evidence is sanitized"); let out = PathBuf::from(std::env::var("GF_G500_CERT_EVIDENCE_OUT").expect("evidence output")); fs::write(out, serde_json::to_vec_pretty(&evidence).unwrap()) .expect("write certification evidence"); diff --git a/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json b/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json index dbdd8cf0b..9a25b6f65 100644 --- a/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json +++ b/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json @@ -1,8 +1,8 @@ { "contractVersion": 1, "rustManifest": "../../../tests/contracts/non-cypher-rust-surface.json", - "releaseSurfaceCount": 257, - "releaseSurfaceDigest": "bb7d3f89204eb3115fd425081d45e874251a20de9759355e867d87e005063219", + "releaseSurfaceCount": 258, + "releaseSurfaceDigest": "6963ebeb6999197aff6de3dbb996c72cbc41561dd2d7400ae81e6d296f8165a7", "requiredEquivalent": [ "GraphForge.adopt_ontology", "GraphForge.clear_ontology", diff --git a/crates/graphforge-bindings-py/tests/non_cypher_release.py b/crates/graphforge-bindings-py/tests/non_cypher_release.py index 1d46e3f19..9a9f8c8c1 100644 --- a/crates/graphforge-bindings-py/tests/non_cypher_release.py +++ b/crates/graphforge-bindings-py/tests/non_cypher_release.py @@ -23,8 +23,8 @@ RUST_MANIFEST = ROOT / "tests/contracts/non-cypher-rust-surface.json" RUST_GATE = ROOT / "scripts/ci/non-cypher-surface-gate.py" PYO3_SOURCE = ROOT / "crates/graphforge-bindings-py/src/lib.rs" -EXPECTED_RUST_DIGEST = "71c92e47b6e43da553be7288724d522e303da5d4677e60f51e7ea8d9f8a41b99" -EXPECTED_RELEASE_DIGEST = "bb7d3f89204eb3115fd425081d45e874251a20de9759355e867d87e005063219" +EXPECTED_RUST_DIGEST = "8e3a0711619a5e50231bf510a76328ea44b7706dfd28c524b760c6564b805bc3" +EXPECTED_RELEASE_DIGEST = "6963ebeb6999197aff6de3dbb996c72cbc41561dd2d7400ae81e6d296f8165a7" PYTHON_ONLY_METHODS = frozenset( { @@ -257,7 +257,7 @@ def _classification_report() -> dict[str, object]: for group in manifest["method_evidence_groups"].values() for method_id in group["ids"] } - assert len(release_methods) == 257 + assert len(release_methods) == 258 assert _digest(release_methods) == EXPECTED_RELEASE_DIGEST assert set(EVIDENCE) == set(manifest["method_evidence_groups"]) diff --git a/crates/graphforge-filesystem/Cargo.toml b/crates/graphforge-filesystem/Cargo.toml index 173a34347..434ae3b27 100644 --- a/crates/graphforge-filesystem/Cargo.toml +++ b/crates/graphforge-filesystem/Cargo.toml @@ -20,6 +20,7 @@ windows-sys = { version = "0.61", features = [ "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_IO", + "Win32_System_Ioctl", ] } [dev-dependencies] diff --git a/crates/graphforge-filesystem/src/lib.rs b/crates/graphforge-filesystem/src/lib.rs index e6f85420d..c2aadd380 100644 --- a/crates/graphforge-filesystem/src/lib.rs +++ b/crates/graphforge-filesystem/src/lib.rs @@ -100,6 +100,15 @@ impl io::Seek for WindowsLegacyCasAdopter { } } +/// Logical and physically allocated byte counts for one retained file handle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileSpaceUsage { + /// Logical end-of-file length visible to readers. + pub logical_bytes: u64, + /// Physical filesystem allocation charged to the file. + pub allocated_bytes: u64, +} + /// Retained directory capability whose children are opened without following /// links or reparse points. #[derive(Debug)] @@ -1092,6 +1101,15 @@ pub fn file_identity(file: &File) -> io::Result { file_identity_platform(file) } +/// Return logical and physically allocated bytes for a retained regular-file handle. +/// +/// The descriptor is the sole authority: this function never resolves or reopens a +/// pathname. Unsupported platforms and native values that cannot be represented +/// safely fail closed. +pub fn file_space_usage(file: &File) -> io::Result { + file_space_usage_platform(file) +} + /// Return the stable native volume/file identity of a non-followed path. pub fn path_identity(path: &Path) -> io::Result { path_identity_platform(path) @@ -1223,6 +1241,15 @@ fn verify_regular_metadata(metadata: &std::fs::Metadata) -> io::Result<()> { Ok(()) } +fn verify_space_usage_metadata(metadata: &std::fs::Metadata) -> io::Result<()> { + if is_link_or_reparse(metadata) || !metadata.is_file() { + return Err(io::Error::other( + "space usage handle is not a regular non-reparse file", + )); + } + Ok(()) +} + #[cfg(windows)] fn is_link_or_reparse(metadata: &std::fs::Metadata) -> bool { use std::os::windows::fs::MetadataExt as _; @@ -1333,6 +1360,22 @@ fn file_identity_platform(file: &File) -> io::Result { unix_identity(file) } +#[cfg(unix)] +fn file_space_usage_platform(file: &File) -> io::Result { + use std::os::unix::fs::MetadataExt as _; + + let metadata = file.metadata()?; + verify_space_usage_metadata(&metadata)?; + let allocated_bytes = metadata + .blocks() + .checked_mul(512) + .ok_or_else(|| io::Error::other("allocated file byte count overflowed u64"))?; + Ok(FileSpaceUsage { + logical_bytes: metadata.len(), + allocated_bytes, + }) +} + #[cfg(unix)] fn path_identity_platform(path: &Path) -> io::Result { use std::os::unix::fs::MetadataExt as _; @@ -1368,6 +1411,11 @@ fn file_identity_platform(file: &File) -> io::Result { windows::file_identity(file) } +#[cfg(windows)] +fn file_space_usage_platform(file: &File) -> io::Result { + windows::file_space_usage(file) +} + #[cfg(windows)] fn path_identity_platform(path: &Path) -> io::Result { windows::identity(path) @@ -1499,6 +1547,14 @@ fn file_identity_platform(_file: &File) -> io::Result { )) } +#[cfg(all(not(unix), not(windows)))] +fn file_space_usage_platform(_file: &File) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "allocated-byte measurement is unsupported on this platform", + )) +} + #[cfg(all(not(unix), not(windows)))] fn path_identity_platform(_path: &Path) -> io::Result { Err(io::Error::new( @@ -1559,17 +1615,18 @@ mod windows { FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE, FILE_DISPOSITION_INFO_EX, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_FLAG_WRITE_THROUGH, FILE_ID_INFO, FILE_NAME_NORMALIZED, FILE_READ_ATTRIBUTES, FILE_RENAME_INFO, - FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_ATTRIBUTES, FileBasicInfo, - FileDispositionInfoEx, FileIdInfo, FileRenameInfo, FileRenameInfoEx, GetDriveTypeW, - GetFileInformationByHandle, GetFileInformationByHandleEx, GetFinalPathNameByHandleW, - GetVolumeInformationW, GetVolumePathNameW, SetFileInformationByHandle, VOLUME_NAME_DOS, + FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_STANDARD_INFO, + FILE_WRITE_ATTRIBUTES, FileBasicInfo, FileDispositionInfoEx, FileIdInfo, FileRenameInfo, + FileRenameInfoEx, FileStandardInfo, GetDriveTypeW, GetFileInformationByHandle, + GetFileInformationByHandleEx, GetFinalPathNameByHandleW, GetVolumeInformationW, + GetVolumePathNameW, SetFileInformationByHandle, VOLUME_NAME_DOS, }; #[cfg(test)] use super::classify_failed_replacement; use super::{ - FileIdentity, ReplaceFileError, WindowsVolumeInformation, is_link_or_reparse, - verify_regular_metadata, + FileIdentity, FileSpaceUsage, ReplaceFileError, WindowsVolumeInformation, + is_link_or_reparse, verify_regular_metadata, verify_space_usage_metadata, }; const DRIVE_FIXED: u32 = 3; @@ -2310,7 +2367,7 @@ mod windows { } fn verify_open_regular(file: &File) -> io::Result<()> { - verify_regular_metadata(&file.metadata()?)?; + verify_space_usage_metadata(&file.metadata()?)?; let information = information(file)?; if information.nNumberOfLinks != 1 { return Err(io::Error::other("replacement path is hard linked")); @@ -2352,6 +2409,33 @@ mod windows { }) } + pub(super) fn file_space_usage(file: &File) -> io::Result { + verify_regular_metadata(&file.metadata()?)?; + let mut information = FILE_STANDARD_INFO::default(); + // SAFETY: the retained handle remains live and the output buffer has + // exactly the size required by FileStandardInfo. + let succeeded = unsafe { + GetFileInformationByHandleEx( + file.as_raw_handle(), + FileStandardInfo, + (&raw mut information).cast(), + u32::try_from(std::mem::size_of::()) + .expect("FILE_STANDARD_INFO size fits u32"), + ) + }; + if succeeded == 0 { + return Err(io::Error::last_os_error()); + } + let logical_bytes = u64::try_from(information.EndOfFile) + .map_err(|_| io::Error::other("native logical file length was negative"))?; + let allocated_bytes = u64::try_from(information.AllocationSize) + .map_err(|_| io::Error::other("native allocated file length was negative"))?; + Ok(FileSpaceUsage { + logical_bytes, + allocated_bytes, + }) + } + pub(super) fn link_count(file: &File) -> io::Result { Ok(u64::from(information(file)?.nNumberOfLinks)) } @@ -2745,6 +2829,95 @@ mod windows { mod tests { use super::*; + #[cfg(windows)] + #[allow(unsafe_code)] + fn mark_sparse(file: &File) { + use std::os::windows::io::AsRawHandle as _; + use windows_sys::Win32::System::IO::DeviceIoControl; + use windows_sys::Win32::System::Ioctl::FSCTL_SET_SPARSE; + + let mut returned = 0; + // SAFETY: `file` retains a live file handle; this control code has no + // input or output buffer, and `returned` remains live for the call. + let succeeded = unsafe { + DeviceIoControl( + file.as_raw_handle(), + FSCTL_SET_SPARSE, + std::ptr::null(), + 0, + std::ptr::null_mut(), + 0, + &raw mut returned, + std::ptr::null_mut(), + ) + }; + assert_ne!(succeeded, 0, "{}", io::Error::last_os_error()); + } + + #[cfg(unix)] + fn mark_sparse(_file: &File) {} + + #[cfg(any(unix, windows))] + #[test] + fn retained_handle_reports_sparse_logical_and_allocated_bytes() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sparse.bin"); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + mark_sparse(&file); + file.set_len(64 * 1024 * 1024).unwrap(); + file.sync_all().unwrap(); + + let usage = file_space_usage(&file).unwrap(); + assert_eq!(usage.logical_bytes, 64 * 1024 * 1024); + assert!( + usage.allocated_bytes < usage.logical_bytes, + "sparse allocation must be physical, not a logical-length proxy: {usage:?}" + ); + } + + #[cfg(any(unix, windows))] + #[test] + fn retained_hard_link_handles_share_identity_and_space_usage() { + let directory = tempfile::tempdir().unwrap(); + let source_path = directory.path().join("source.bin"); + let alias_path = directory.path().join("alias.bin"); + let source = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&source_path) + .unwrap(); + mark_sparse(&source); + source.set_len(32 * 1024 * 1024).unwrap(); + source.sync_all().unwrap(); + std::fs::hard_link(&source_path, &alias_path).unwrap(); + let alias = File::open(&alias_path).unwrap(); + + assert_eq!( + file_identity(&source).unwrap(), + file_identity(&alias).unwrap() + ); + assert_eq!( + file_space_usage(&source).unwrap(), + file_space_usage(&alias).unwrap() + ); + + std::fs::remove_file(&source_path).unwrap(); + assert_eq!( + file_identity(&source).unwrap(), + file_identity(&alias).unwrap() + ); + assert_eq!( + file_space_usage(&source).unwrap(), + file_space_usage(&alias).unwrap() + ); + } + #[cfg(unix)] const FIFO_CHILD_ENV: &str = "GRAPHFORGE_FILESYSTEM_FIFO_CHILD"; diff --git a/crates/graphforge-storage/src/graph_construction.rs b/crates/graphforge-storage/src/graph_construction.rs index d5e84de59..3b62851d7 100644 --- a/crates/graphforge-storage/src/graph_construction.rs +++ b/crates/graphforge-storage/src/graph_construction.rs @@ -37,7 +37,7 @@ use uuid::Uuid; use crate::UuidIndexKind; use crate::uuid_membership::{AuthenticatedUuidIndexSnapshot, UuidConstructionSnapshotWork}; -const FORMAT_VERSION: u32 = 5; +const FORMAT_VERSION: u32 = 6; const PRIVATE_ROOT: &str = ".graphforge-construction"; const SESSION_LOCK: &str = "session.lock"; const CHECKPOINT: &str = "checkpoint.json"; @@ -46,7 +46,10 @@ const SHAPE_INTENT: &str = "shape-intent.json"; const PUBLICATION_INTENT: &str = "publication-intent.json"; const PUBLICATION_RECEIPT: &str = "publication-receipt.json"; const BLOCK_BYTES: usize = 1 << 20; -const MAX_CONTROL_BYTES: u64 = 64 << 10; +// Checkpoints include bounded authenticated allocation identities. The supported +// heterogeneous-schema cardinality can legitimately exceed 64 KiB while still +// remaining far below the separately bounded shape inventory. +const MAX_CONTROL_BYTES: u64 = 1 << 20; const MAX_SHAPE_CONTROL_BYTES: u64 = 32 << 20; const IDENTITY_WIDTH: usize = 16; const ENDPOINT_WIDTH: usize = 48; @@ -313,21 +316,92 @@ pub struct GraphConstructionEvidence { /// All application-observed shaped payload bytes read by canonical encoding. #[serde(default)] pub encode_application_read_bytes: u64, + /// Actual non-empty reads performed by canonical encoding. + #[serde(default)] + pub encode_application_read_operations: u64, + /// Payload bytes submitted by canonical artifact writers. + #[serde(default)] + pub encode_application_write_bytes: u64, + /// Actual canonical artifact write submissions. + #[serde(default)] + pub encode_application_write_operations: u64, + /// Canonical encoding file and directory durability barriers. + #[serde(default)] + pub encode_fsync_operations: u64, /// Application-observed durable control bytes read immediately before publication. #[serde(default)] pub publication_application_read_bytes: u64, + /// Actual non-empty durable-control reads immediately before publication. + #[serde(default)] + pub publication_application_read_operations: u64, /// Application-observed source or reused-object payload bytes read by CAS adoption. #[serde(default)] pub cas_application_read_bytes: u64, + /// Actual non-empty CAS source/authentication reads. + #[serde(default)] + pub cas_application_read_operations: u64, + /// Payload bytes submitted to CAS temporary-object writers. + #[serde(default)] + pub cas_application_write_bytes: u64, + /// Actual CAS temporary-object write submissions. + #[serde(default)] + pub cas_application_write_operations: u64, + /// CAS file and directory durability barriers. + #[serde(default)] + pub cas_fsync_operations: u64, /// Application-observed published payload bytes read while hydrating the workspace. #[serde(default)] pub hydration_application_read_bytes: u64, + /// Actual non-empty reads performed during hydration and verification. + #[serde(default)] + pub hydration_application_read_operations: u64, + /// Payload bytes submitted to hydrated workspace writers. + #[serde(default)] + pub hydration_application_write_bytes: u64, + /// Actual hydrated workspace write submissions. + #[serde(default)] + pub hydration_application_write_operations: u64, + /// Hydration file and directory durability barriers. + #[serde(default)] + pub hydration_fsync_operations: u64, + /// Artifact bytes authenticated while repairing an interrupted append. + #[serde(default)] + pub recovery_application_read_bytes: u64, + /// Bounded artifact reads used by interrupted-append recovery. + #[serde(default)] + pub recovery_application_read_operations: u64, /// New canonical graph payload bytes emitted by encoding. #[serde(default)] pub canonical_output_bytes: u64, /// Staged artifact bytes plus structurally retained parent payload bytes. #[serde(default)] pub staged_and_retained_disk_bytes: u64, + /// Receipt-derived retained construction artifacts by semantic category. + /// Persisted in the checkpoint so resume never scans the session tree. + #[serde(default)] + pub storage_current: BTreeMap, + /// Per-category peak allocated bytes observed as receipt-backed staging + /// artifacts accumulated. These are transient construction bytes, not + /// committed-generation allocation. + #[serde(default)] + pub storage_transient_peak_allocated_bytes: BTreeMap, + /// High-water mark of the union of all simultaneously retained, + /// receipt-authenticated construction artifacts. Unlike the per-category + /// diagnostics above, this is a total and categories are never treated as + /// mutually exclusive. + #[serde(default)] + pub storage_transient_peak_total_allocated_bytes: u64, + /// Exact currently retained construction allocation keyed by authenticated + /// native `(volume, file-id)` identity. This is persisted with the + /// checkpoint so lifecycle qualification can union it with other owners + /// without double counting aliases. + #[serde(default)] + pub storage_active_identity_allocated_bytes: BTreeMap, + /// Writer-owned identity deltas in exact operation order. Unlike the final + /// active map, this preserves staging/merge/encoding coexistence for files + /// removed before construction returns. + #[serde(default)] + pub storage_allocation_transitions: Vec, /// Rows accepted. pub input_rows: u64, /// Non-replay chunks accepted. @@ -384,8 +458,14 @@ pub struct GraphConstructionEvidence { pub replayed_chunks: u64, /// Temporary and final fixed-width records read by canonical shaping. pub merge_read_records: u64, + /// Actual non-empty fixed-run read submissions completed by shaping. + #[serde(default)] + pub merge_read_operations: u64, /// Temporary and final fixed-width records written by canonical shaping. pub merge_written_records: u64, + /// Actual non-empty fixed-run write submissions completed by shaping. + #[serde(default)] + pub merge_write_operations: u64, /// External merge groups completed (including intermediate levels). pub merge_groups: u64, /// Highest number of simultaneously open merge inputs. @@ -404,6 +484,10 @@ pub struct GraphConstructionEvidence { pub merge_passes: u64, /// Largest measured temporary merge footprint. pub peak_merge_temporary_bytes: u64, + /// Currently retained filesystem allocation owned by authenticated + /// shape/merge artifacts. + #[serde(default)] + pub current_merge_temporary_allocated_bytes: u64, /// Largest explicitly retained application buffer set during append. This /// includes input Arrow buffers, extracted fixed runs, sorted Arrow output, /// and a conservative full-batch Parquet encoding window; allocator/RSS is @@ -441,6 +525,7 @@ impl GraphConstructionEvidence { .saturating_add(self.publication_application_read_bytes) .saturating_add(self.cas_application_read_bytes) .saturating_add(self.hydration_application_read_bytes) + .saturating_add(self.recovery_application_read_bytes) } } @@ -562,6 +647,7 @@ impl IdentityRecord { pub(crate) struct ArtifactReceipt { name: String, bytes: u64, + allocated_bytes: u64, sha256: String, identity: IdentityRecord, write_operations: u64, @@ -841,7 +927,7 @@ impl GraphConstructionSession { validate_publication_intent(&intent, &self.checkpoint)?; install_control(&self.root, PUBLICATION_INTENT, &intent)?; self.checkpoint.publication_state = Some(ConstructionPublicationState::Publishing); - replace_control(&self.root, CHECKPOINT, &self.checkpoint)?; + replace_checkpoint_control(&self.root, &self.checkpoint)?; Ok(intent) } @@ -891,7 +977,7 @@ impl GraphConstructionSession { let receipt = provisional; install_control(&self.root, PUBLICATION_RECEIPT, &receipt)?; self.checkpoint.publication_state = Some(ConstructionPublicationState::Published); - replace_control(&self.root, CHECKPOINT, &self.checkpoint)?; + replace_checkpoint_control(&self.root, &self.checkpoint)?; Ok(receipt) } @@ -945,12 +1031,46 @@ impl GraphConstructionSession { self.checkpoint.budgets, &mut cancelled, )?; + record_encoded_active_artifacts(&self.root, &encoded, &mut self.checkpoint.evidence)?; self.checkpoint.evidence.encode_application_read_bytes = self .checkpoint .evidence .encode_application_read_bytes .saturating_add(encoded.evidence.input_read_bytes) .saturating_add(encoded.evidence.membership_read_bytes); + self.checkpoint.evidence.encode_application_read_operations = self + .checkpoint + .evidence + .encode_application_read_operations + .saturating_add(encoded.evidence.input_read_operations) + .saturating_add(encoded.evidence.membership_read_operations) + .saturating_add(encoded.evidence.source_spool_read_operations); + self.checkpoint.evidence.encode_application_write_bytes = self + .checkpoint + .evidence + .encode_application_write_bytes + .saturating_add(encoded.evidence.output_write_bytes) + .saturating_add(encoded.evidence.membership_total_write_bytes) + .saturating_add(encoded.evidence.source_spool_write_bytes) + .saturating_add(encoded.evidence.ordinal_artifact_write_bytes) + .saturating_add(encoded.evidence.ordinal_publication_write_bytes); + self.checkpoint.evidence.encode_application_write_operations = self + .checkpoint + .evidence + .encode_application_write_operations + .saturating_add(encoded.evidence.output_write_operations) + .saturating_add(encoded.evidence.membership_write_operations) + .saturating_add(encoded.evidence.source_spool_write_operations) + .saturating_add(encoded.evidence.ordinal_artifact_write_operations) + .saturating_add(encoded.evidence.ordinal_publication_write_operations); + self.checkpoint.evidence.encode_fsync_operations = self + .checkpoint + .evidence + .encode_fsync_operations + .saturating_add(encoded.evidence.fsync_operations) + .saturating_add(encoded.evidence.membership_fsync_operations) + .saturating_add(encoded.evidence.source_spool_fsync_operations) + .saturating_add(encoded.evidence.ordinal_fsync_operations); self.checkpoint.evidence.canonical_output_bytes = encoded .artifacts .iter() @@ -975,7 +1095,7 @@ impl GraphConstructionSession { Some(_) => {} None => { self.checkpoint.encoding_inventory_sha256 = Some(inventory_authority); - replace_control(&self.root, CHECKPOINT, &self.checkpoint)?; + replace_checkpoint_control(&self.root, &self.checkpoint)?; } } Ok(encoded) @@ -1050,7 +1170,7 @@ impl GraphConstructionSession { if encoding.generation != self.checkpoint.parent_topology_generation.saturating_add(1) { return Err(storage("publication topology generation changed")); } - let inventory_control_bytes = + let inventory_control = crate::graph_construction_encoding::authenticate_inventory_control_for_publication( &self.root, encoding, )?; @@ -1058,7 +1178,14 @@ impl GraphConstructionSession { .checkpoint .evidence .publication_application_read_bytes - .saturating_add(inventory_control_bytes); + .saturating_add(inventory_control.read_bytes); + self.checkpoint + .evidence + .publication_application_read_operations = self + .checkpoint + .evidence + .publication_application_read_operations + .saturating_add(inventory_control.read_calls); let admission = crate::filesystem_admission::admit_project_lifecycle( &self.project_path, self.checkpoint.lifecycle_mode, @@ -1149,6 +1276,26 @@ impl GraphConstructionSession { .evidence .cas_application_read_bytes .saturating_add(cas_evidence.payload_bytes_hashed); + self.checkpoint.evidence.cas_application_read_operations = self + .checkpoint + .evidence + .cas_application_read_operations + .saturating_add(cas_evidence.read_calls); + self.checkpoint.evidence.cas_application_write_bytes = self + .checkpoint + .evidence + .cas_application_write_bytes + .saturating_add(cas_evidence.write_bytes); + self.checkpoint.evidence.cas_application_write_operations = self + .checkpoint + .evidence + .cas_application_write_operations + .saturating_add(cas_evidence.write_calls); + self.checkpoint.evidence.cas_fsync_operations = self + .checkpoint + .evidence + .cas_fsync_operations + .saturating_add(cas_evidence.fsync_calls); if graphforge_filesystem::path_identity(&workspace).map_err(storage)? != encoded_directory.identity() { @@ -1723,13 +1870,37 @@ impl GraphConstructionSession { }; recover_shape_intent(&session.root, &mut session.checkpoint)?; session.recover_intent()?; + if session + .checkpoint + .evidence + .storage_allocation_transitions + .is_empty() + && !session + .checkpoint + .evidence + .storage_active_identity_allocated_bytes + .is_empty() + { + session + .checkpoint + .evidence + .storage_allocation_transitions + .push(crate::StorageAllocationTransition { + installed: session + .checkpoint + .evidence + .storage_active_identity_allocated_bytes + .clone(), + removed: BTreeSet::new(), + }); + } session.revalidate_authority()?; if session.checkpoint.next_sequence != 0 && session.checkpoint.evidence.immutable_artifacts == 0 { session.checkpoint.evidence.immutable_artifacts = authenticated_receipt_artifact_count(&session.root, &session.checkpoint)?; - replace_control(&session.root, CHECKPOINT, &session.checkpoint)?; + replace_checkpoint_control(&session.root, &session.checkpoint)?; } Ok(session) } @@ -1830,16 +2001,43 @@ impl GraphConstructionSession { &self.checkpoint.evidence } - /// Record application-observed reads performed by the facade's post-publication - /// hydration before the refreshed workspace becomes visible. + /// Record storage-owned application I/O performed by the facade's + /// post-publication hydration before the refreshed workspace becomes visible. #[doc(hidden)] - pub fn record_hydration_application_read_bytes(&mut self, bytes: u64) -> Result<(), GfError> { + pub fn record_hydration_evidence( + &mut self, + hydration: &crate::GraphFilesOpenEvidence, + ) -> Result<(), GfError> { self.checkpoint.evidence.hydration_application_read_bytes = self .checkpoint .evidence .hydration_application_read_bytes - .saturating_add(bytes); - replace_control(&self.root, CHECKPOINT, &self.checkpoint) + .saturating_add(hydration.application_read_bytes); + self.checkpoint + .evidence + .hydration_application_read_operations = self + .checkpoint + .evidence + .hydration_application_read_operations + .saturating_add(hydration.application_read_calls); + self.checkpoint.evidence.hydration_application_write_bytes = self + .checkpoint + .evidence + .hydration_application_write_bytes + .saturating_add(hydration.application_write_bytes); + self.checkpoint + .evidence + .hydration_application_write_operations = self + .checkpoint + .evidence + .hydration_application_write_operations + .saturating_add(hydration.application_write_calls); + self.checkpoint.evidence.hydration_fsync_operations = self + .checkpoint + .evidence + .hydration_fsync_operations + .saturating_add(hydration.fsync_calls); + replace_checkpoint_control(&self.root, &self.checkpoint) } /// Number of durably accepted chunks. @@ -1930,7 +2128,7 @@ impl GraphConstructionSession { .saturating_add(work.operations); self.checkpoint.evidence.replayed_chunks = self.checkpoint.evidence.replayed_chunks.saturating_add(1); - replace_control(&self.root, CHECKPOINT, &self.checkpoint)?; + replace_checkpoint_control(&self.root, &self.checkpoint)?; return Ok(receipt); } return Err(storage("conflicting construction chunk replay")); @@ -2115,7 +2313,7 @@ impl GraphConstructionSession { .saturating_add(read_bytes); self.checkpoint.state = GraphConstructionState::Sealed; self.checkpoint.publication_state = Some(ConstructionPublicationState::Sealed); - replace_control(&self.root, CHECKPOINT, &self.checkpoint) + replace_checkpoint_control(&self.root, &self.checkpoint) } /// Validate the sealed identity domains and produce deterministic, @@ -2407,11 +2605,6 @@ impl GraphConstructionSession { &mut cancelled, &mut self.checkpoint.evidence, )?; - self.checkpoint.evidence.peak_merge_temporary_bytes = self - .checkpoint - .evidence - .peak_merge_temporary_bytes - .max(measured_shape_bytes(&self.root)?); let shape = ConstructionShape { ontology_mode: self.checkpoint.ontology_mode, semantic_authority_sha256: self.checkpoint.semantic_authority_sha256.clone(), @@ -2496,7 +2689,7 @@ impl GraphConstructionSession { }, )?; construction_failpoint("shape.after_complete_inventory"); - replace_control(&self.root, CHECKPOINT, &self.checkpoint)?; + replace_checkpoint_control(&self.root, &self.checkpoint)?; construction_failpoint("shape.after_evidence_checkpoint"); Ok(shape) } @@ -2509,7 +2702,7 @@ impl GraphConstructionSession { return Err(storage("non-staging session belongs to the publisher")); } self.checkpoint.state = GraphConstructionState::Aborted; - replace_control(&self.root, CHECKPOINT, &self.checkpoint) + replace_checkpoint_control(&self.root, &self.checkpoint) } #[allow(clippy::too_many_lines)] @@ -2529,7 +2722,19 @@ impl GraphConstructionSession { if receipt != receipt_from_intent(&intent)? { return Err(storage("recovered receipt differs from durable intent")); } - validate_receipt_artifacts(&self.root, &receipt)?; + let recovery_work = validate_receipt_artifacts(&self.root, &receipt)?; + self.checkpoint.evidence.recovery_application_read_bytes = self + .checkpoint + .evidence + .recovery_application_read_bytes + .saturating_add(recovery_work.bytes); + self.checkpoint + .evidence + .recovery_application_read_operations = self + .checkpoint + .evidence + .recovery_application_read_operations + .saturating_add(recovery_work.operations); let body = serde_json::to_vec(&receipt).map_err(storage)?; if intent.sequence < self.checkpoint.next_sequence && self.checkpoint.last_receipt_sha256.as_deref() @@ -2575,7 +2780,19 @@ impl GraphConstructionSession { .into_iter() .flatten() { - authenticate_artifact(&self.root, &artifact)?; + let recovery_work = authenticate_artifact(&self.root, &artifact)?; + self.checkpoint.evidence.recovery_application_read_bytes = self + .checkpoint + .evidence + .recovery_application_read_bytes + .saturating_add(recovery_work.bytes); + self.checkpoint + .evidence + .recovery_application_read_operations = self + .checkpoint + .evidence + .recovery_application_read_operations + .saturating_add(recovery_work.operations); unlink_artifact(&self.root, &artifact)?; } let stem = artifact_stem(intent.sequence, intent.kind); @@ -2665,11 +2882,24 @@ impl GraphConstructionSession { evidence.peak_accounted_live_bytes = evidence .peak_accounted_live_bytes .max(receipt.accounted_live_bytes); + // Chunk receipts describe private construction inputs. They are not + // canonical topology until shaping, encoding, and generation-last + // publication succeed, so attribution must keep them in staging. for artifact in [&receipt.parquet, &receipt.identities, &receipt.details] .into_iter() .chain(receipt.endpoints.iter()) { evidence.immutable_artifacts = evidence.immutable_artifacts.saturating_add(1); + let identity_key = format!( + "{:016x}:{}", + artifact.identity.volume_serial, artifact.identity.file_id + ); + record_active_identity_install( + evidence, + identity_key, + artifact.allocated_bytes, + "construction artifact identity was already active", + )?; evidence.write_bytes = evidence.write_bytes.saturating_add(artifact.bytes); evidence.write_operations = evidence .write_operations @@ -2677,6 +2907,32 @@ impl GraphConstructionSession { evidence.fsync_operations = evidence .fsync_operations .saturating_add(artifact.fsync_operations); + let totals = evidence + .storage_current + .entry(crate::ArtifactCategory::ConstructionStaging) + .or_default(); + totals.logical_references = totals.logical_references.saturating_add(1); + totals.logical_bytes = totals.logical_bytes.saturating_add(artifact.bytes); + totals.physical_objects = totals.physical_objects.saturating_add(1); + totals.physical_logical_bytes = + totals.physical_logical_bytes.saturating_add(artifact.bytes); + totals.allocated_bytes = totals + .allocated_bytes + .saturating_add(artifact.allocated_bytes); + evidence + .storage_transient_peak_allocated_bytes + .entry(crate::ArtifactCategory::ConstructionStaging) + .and_modify(|peak| *peak = (*peak).max(totals.allocated_bytes)) + .or_insert(totals.allocated_bytes); + let current_union = evidence + .storage_current + .values() + .fold(0_u64, |total, item| { + total.saturating_add(item.allocated_bytes) + }); + evidence.storage_transient_peak_total_allocated_bytes = evidence + .storage_transient_peak_total_allocated_bytes + .max(current_union); } self.checkpoint.next_sequence = self.checkpoint.next_sequence.saturating_add(1); self.checkpoint.saw_edge |= receipt.kind == ConstructionChunkKind::Edge; @@ -2693,7 +2949,7 @@ impl GraphConstructionSession { } } self.checkpoint.last_receipt_sha256 = Some(sha256(receipt_bytes)); - replace_control(&self.root, CHECKPOINT, &self.checkpoint) + replace_checkpoint_control(&self.root, &self.checkpoint) } fn revalidate_authority(&self) -> Result<(), GfError> { @@ -3210,6 +3466,9 @@ fn write_parquet( let receipt = ArtifactReceipt { name: name.to_owned(), bytes: hashing.bytes, + allocated_bytes: graphforge_filesystem::file_space_usage(&hashing.inner) + .map_err(storage)? + .allocated_bytes, sha256: hex(&hashing.digest.clone().finalize()), identity: identity.into(), write_operations: hashing.operations, @@ -3250,6 +3509,9 @@ fn write_fixed_run( let receipt = ArtifactReceipt { name: name.to_owned(), bytes: writer.bytes, + allocated_bytes: graphforge_filesystem::file_space_usage(&writer.inner) + .map_err(storage)? + .allocated_bytes, sha256: hex(&writer.digest.finalize()), identity: identity.into(), write_operations: writer.operations, @@ -3318,38 +3580,21 @@ fn recover_shape_intent( .final_evidence .as_ref() .ok_or_else(|| storage("complete shape manifest lacks final evidence"))?; - if checkpoint.evidence == intent.baseline_evidence - && checkpoint.shape_authority_sha256.is_none() - { - checkpoint.evidence = final_evidence.clone(); - checkpoint.shape_authority_sha256 = Some(expected_shape_authority); - replace_control(root, CHECKPOINT, checkpoint)?; - } else { - let mut shape_owned_evidence = checkpoint.evidence.clone(); - shape_owned_evidence.encode_application_read_bytes = - final_evidence.encode_application_read_bytes; - shape_owned_evidence.publication_application_read_bytes = - final_evidence.publication_application_read_bytes; - shape_owned_evidence.cas_application_read_bytes = - final_evidence.cas_application_read_bytes; - shape_owned_evidence.hydration_application_read_bytes = - final_evidence.hydration_application_read_bytes; - shape_owned_evidence.canonical_output_bytes = final_evidence.canonical_output_bytes; - shape_owned_evidence.staged_and_retained_disk_bytes = - final_evidence.staged_and_retained_disk_bytes; - if shape_owned_evidence == *final_evidence - && checkpoint.shape_authority_sha256.as_deref() == Some(&expected_shape_authority) - { - return Ok(()); - } - return Err(storage("shape evidence authority differs from inventory")); - } + recover_final_shape_evidence( + root, + checkpoint, + &intent.baseline_evidence, + final_evidence, + expected_shape_authority, + )?; return Ok(()); } if intent.shape.is_some() || !intent.outputs.is_empty() { return Err(storage("incomplete shape intent claims completed output")); } - if intent.final_evidence.is_some() || checkpoint.evidence != intent.baseline_evidence { + if intent.final_evidence.is_some() + || !persisted_evidence_equivalent(&checkpoint.evidence, &intent.baseline_evidence) + { return Err(storage("incomplete shape changed committed evidence")); } cleanup_incomplete_shape_capabilities(root)?; @@ -3373,6 +3618,75 @@ fn recover_shape_intent( unlink_named(root, SHAPE_INTENT) } +fn recover_final_shape_evidence( + root: &StableDirectory, + checkpoint: &mut Checkpoint, + baseline: &GraphConstructionEvidence, + final_evidence: &GraphConstructionEvidence, + expected_authority: String, +) -> Result<(), GfError> { + if persisted_evidence_equivalent(&checkpoint.evidence, baseline) + && checkpoint.shape_authority_sha256.is_none() + { + checkpoint.evidence = final_evidence.clone(); + checkpoint.shape_authority_sha256 = Some(expected_authority); + return replace_checkpoint_control(root, checkpoint); + } + let mut observed = checkpoint.evidence.clone(); + copy_post_shape_io(&mut observed, final_evidence); + if observed == *final_evidence + && checkpoint.shape_authority_sha256.as_deref() == Some(&expected_authority) + { + return Ok(()); + } + Err(storage("shape evidence authority differs from inventory")) +} + +fn persisted_evidence_equivalent( + left: &GraphConstructionEvidence, + right: &GraphConstructionEvidence, +) -> bool { + let mut left = left.clone(); + let mut right = right.clone(); + left.storage_allocation_transitions.clear(); + right.storage_allocation_transitions.clear(); + left == right +} + +fn copy_post_shape_io(target: &mut GraphConstructionEvidence, source: &GraphConstructionEvidence) { + target.storage_current = source.storage_current.clone(); + target.storage_transient_peak_allocated_bytes = + source.storage_transient_peak_allocated_bytes.clone(); + target.storage_transient_peak_total_allocated_bytes = + source.storage_transient_peak_total_allocated_bytes; + target.storage_active_identity_allocated_bytes = + source.storage_active_identity_allocated_bytes.clone(); + target + .storage_allocation_transitions + .clone_from(&source.storage_allocation_transitions); + target.current_merge_temporary_allocated_bytes = source.current_merge_temporary_allocated_bytes; + target.peak_merge_temporary_bytes = source.peak_merge_temporary_bytes; + target.encode_application_read_bytes = source.encode_application_read_bytes; + target.encode_application_read_operations = source.encode_application_read_operations; + target.encode_application_write_bytes = source.encode_application_write_bytes; + target.encode_application_write_operations = source.encode_application_write_operations; + target.encode_fsync_operations = source.encode_fsync_operations; + target.publication_application_read_bytes = source.publication_application_read_bytes; + target.publication_application_read_operations = source.publication_application_read_operations; + target.cas_application_read_bytes = source.cas_application_read_bytes; + target.cas_application_read_operations = source.cas_application_read_operations; + target.cas_application_write_bytes = source.cas_application_write_bytes; + target.cas_application_write_operations = source.cas_application_write_operations; + target.cas_fsync_operations = source.cas_fsync_operations; + target.hydration_application_read_bytes = source.hydration_application_read_bytes; + target.hydration_application_read_operations = source.hydration_application_read_operations; + target.hydration_application_write_bytes = source.hydration_application_write_bytes; + target.hydration_application_write_operations = source.hydration_application_write_operations; + target.hydration_fsync_operations = source.hydration_fsync_operations; + target.canonical_output_bytes = source.canonical_output_bytes; + target.staged_and_retained_disk_bytes = source.staged_and_retained_disk_bytes; +} + fn cleanup_incomplete_shape_capabilities(root: &StableDirectory) -> Result<(), GfError> { for child in root.child_names().map_err(storage)? { let Some(name) = child.to_str() else { continue }; @@ -3569,7 +3883,7 @@ fn recover_publication( && checkpoint.publication_state == Some(ConstructionPublicationState::Sealed) { checkpoint.publication_state = Some(ConstructionPublicationState::Publishing); - replace_control(root, CHECKPOINT, checkpoint)?; + replace_checkpoint_control(root, checkpoint)?; } let receipt_exists = match root.open_child_file(OsStr::new(PUBLICATION_RECEIPT)) { Ok(file) => { @@ -3584,7 +3898,7 @@ fn recover_publication( authenticate_published_target(project_dir, checkpoint, &receipt)?; if checkpoint.publication_state == Some(ConstructionPublicationState::Publishing) { checkpoint.publication_state = Some(ConstructionPublicationState::Published); - replace_control(root, CHECKPOINT, checkpoint)?; + replace_checkpoint_control(root, checkpoint)?; } } match checkpoint.publication_state { @@ -3747,6 +4061,9 @@ fn receipt_for_existing_with_work( ArtifactReceipt { name: name.to_owned(), bytes, + allocated_bytes: graphforge_filesystem::file_space_usage(&file) + .map_err(storage)? + .allocated_bytes, sha256: hex(&digest.finalize()), identity: identity.into(), write_operations: 0, @@ -3896,26 +4213,216 @@ fn is_shape_artifact_name(name: &str) -> bool { }) } -fn measured_shape_bytes(root: &StableDirectory) -> Result { - let mut bytes = 0_u64; - for name in root.child_names().map_err(storage)? { - let Some(name) = name.to_str() else { continue }; - if is_shape_artifact_name(name) { - bytes = bytes.saturating_add( - root.open_child_file(OsStr::new(name)) - .map_err(storage)? - .metadata() - .map_err(storage)? - .len(), - ); +fn record_active_identity_install( + evidence: &mut GraphConstructionEvidence, + identity: String, + allocated_bytes: u64, + duplicate_message: &'static str, +) -> Result<(), GfError> { + if evidence + .storage_active_identity_allocated_bytes + .insert(identity.clone(), allocated_bytes) + .is_some() + { + return Err(storage(duplicate_message)); + } + evidence + .storage_allocation_transitions + .push(crate::StorageAllocationTransition { + installed: BTreeMap::from([(identity, allocated_bytes)]), + removed: BTreeSet::new(), + }); + Ok(()) +} + +fn record_active_identity_remove( + evidence: &mut GraphConstructionEvidence, + identity: &str, +) -> Result { + let removed = evidence + .storage_active_identity_allocated_bytes + .remove(identity) + .ok_or_else(|| storage("active construction identity is absent"))?; + evidence + .storage_allocation_transitions + .push(crate::StorageAllocationTransition { + installed: BTreeMap::new(), + removed: BTreeSet::from([identity.to_owned()]), + }); + Ok(removed) +} + +fn record_shape_artifact_install( + evidence: &mut GraphConstructionEvidence, + receipt: &ArtifactReceipt, +) -> Result<(), GfError> { + let identity_key = format!( + "{:016x}:{}", + receipt.identity.volume_serial, receipt.identity.file_id + ); + record_active_identity_install( + evidence, + identity_key, + receipt.allocated_bytes, + "shape artifact identity installed twice", + )?; + let totals = evidence + .storage_current + .entry(crate::ArtifactCategory::ConstructionStaging) + .or_default(); + totals.logical_references = totals.logical_references.saturating_add(1); + totals.logical_bytes = totals.logical_bytes.saturating_add(receipt.bytes); + totals.physical_objects = totals.physical_objects.saturating_add(1); + totals.physical_logical_bytes = totals.physical_logical_bytes.saturating_add(receipt.bytes); + totals.allocated_bytes = totals + .allocated_bytes + .saturating_add(receipt.allocated_bytes); + evidence.current_merge_temporary_allocated_bytes = evidence + .current_merge_temporary_allocated_bytes + .saturating_add(receipt.allocated_bytes); + evidence.peak_merge_temporary_bytes = evidence + .peak_merge_temporary_bytes + .max(evidence.current_merge_temporary_allocated_bytes); + evidence + .storage_transient_peak_allocated_bytes + .entry(crate::ArtifactCategory::ConstructionStaging) + .and_modify(|peak| *peak = (*peak).max(totals.allocated_bytes)) + .or_insert(totals.allocated_bytes); + let union = evidence + .storage_current + .values() + .fold(0_u64, |total, item| { + total.saturating_add(item.allocated_bytes) + }); + evidence.storage_transient_peak_total_allocated_bytes = evidence + .storage_transient_peak_total_allocated_bytes + .max(union); + Ok(()) +} + +fn record_encoded_active_artifacts( + session_root: &StableDirectory, + encoding: &GraphConstructionEncoding, + evidence: &mut GraphConstructionEvidence, +) -> Result<(), GfError> { + let encoded_root = session_root + .open_child_directory(OsStr::new(&encoding.root)) + .map_err(storage)? + .open_child_directory(OsStr::new("graph")) + .map_err(storage)?; + for artifact in &encoding.artifacts { + let components = Path::new(&artifact.path) + .components() + .map(|component| match component { + std::path::Component::Normal(value) => Ok(value.to_owned()), + _ => Err(storage("encoded artifact path is not normalized")), + }) + .collect::, _>>()?; + let (name, directories) = components + .split_last() + .ok_or_else(|| storage("encoded artifact path is empty"))?; + let mut directory = encoded_root.try_clone().map_err(storage)?; + for child in directories { + directory = directory.open_child_directory(child).map_err(storage)?; + } + let file = directory.open_child_file(name).map_err(storage)?; + let identity = file_identity(&file).map_err(storage)?; + let usage = graphforge_filesystem::file_space_usage(&file).map_err(storage)?; + if usage.logical_bytes != artifact.bytes { + return Err(storage("encoded artifact allocation authority changed")); + } + let identity_key = format!("{:016x}:{}", identity.volume_serial, hex(&identity.file_id)); + if let Some(existing) = evidence + .storage_active_identity_allocated_bytes + .get(&identity_key) + { + if *existing != usage.allocated_bytes { + return Err(storage("encoded artifact identity allocation changed")); + } + continue; } + record_active_identity_install( + evidence, + identity_key, + usage.allocated_bytes, + "encoded artifact identity installed twice", + )?; + let totals = evidence + .storage_current + .entry(crate::ArtifactCategory::ConstructionStaging) + .or_default(); + totals.logical_references = totals.logical_references.saturating_add(1); + totals.logical_bytes = totals.logical_bytes.saturating_add(usage.logical_bytes); + totals.physical_objects = totals.physical_objects.saturating_add(1); + totals.physical_logical_bytes = totals + .physical_logical_bytes + .saturating_add(usage.logical_bytes); + totals.allocated_bytes = totals.allocated_bytes.saturating_add(usage.allocated_bytes); + let active_total = evidence + .storage_active_identity_allocated_bytes + .values() + .try_fold(0_u64, |total, value| total.checked_add(*value)) + .ok_or_else(|| storage("active construction allocation overflow"))?; + evidence.storage_transient_peak_total_allocated_bytes = evidence + .storage_transient_peak_total_allocated_bytes + .max(active_total); } - Ok(bytes) + Ok(()) +} + +fn unlink_shape_artifact( + root: &StableDirectory, + name: &str, + evidence: &mut GraphConstructionEvidence, +) -> Result<(), GfError> { + let receipt = receipt_for_existing(root, name)?; + unlink_artifact(root, &receipt)?; + let identity_key = format!( + "{:016x}:{}", + receipt.identity.volume_serial, receipt.identity.file_id + ); + let removed = record_active_identity_remove(evidence, &identity_key).map_err(|_| { + storage(format!( + "shape active identity ledger is absent for {name} ({identity_key})" + )) + })?; + if removed != receipt.allocated_bytes { + return Err(storage("shape active identity allocation changed")); + } + let totals = evidence + .storage_current + .get_mut(&crate::ArtifactCategory::ConstructionStaging) + .ok_or_else(|| storage("shape allocation ledger is absent"))?; + totals.logical_references = totals + .logical_references + .checked_sub(1) + .ok_or_else(|| storage("shape logical-reference ledger underflow"))?; + totals.logical_bytes = totals + .logical_bytes + .checked_sub(receipt.bytes) + .ok_or_else(|| storage("shape logical-byte ledger underflow"))?; + totals.physical_objects = totals + .physical_objects + .checked_sub(1) + .ok_or_else(|| storage("shape physical-object ledger underflow"))?; + totals.physical_logical_bytes = totals + .physical_logical_bytes + .checked_sub(receipt.bytes) + .ok_or_else(|| storage("shape physical-logical ledger underflow"))?; + totals.allocated_bytes = totals + .allocated_bytes + .checked_sub(receipt.allocated_bytes) + .ok_or_else(|| storage("shape allocated-byte ledger underflow"))?; + evidence.current_merge_temporary_allocated_bytes = evidence + .current_merge_temporary_allocated_bytes + .checked_sub(receipt.allocated_bytes) + .ok_or_else(|| storage("shape active-allocation ledger underflow"))?; + Ok(()) } fn account_merge_read(evidence: &mut GraphConstructionEvidence) { + let _ = N; evidence.merge_read_records = evidence.merge_read_records.saturating_add(1); - evidence.merge_read_bytes = evidence.merge_read_bytes.saturating_add(N as u64); } fn account_merge_write(evidence: &mut GraphConstructionEvidence) { @@ -3939,6 +4446,52 @@ fn account_sequential_write(bytes: u64, evidence: &mut GraphConstructionEvidence } } +fn account_fixed_read_operations( + counter: &IoCounter, + evidence: &mut GraphConstructionEvidence, +) -> Result<(), GfError> { + let (bytes, operations) = counter.values(); + if (bytes == 0) != (operations == 0) { + return Err(storage("fixed-run read bytes and submissions disagree")); + } + evidence.merge_read_bytes = evidence.merge_read_bytes.saturating_add(bytes); + evidence.merge_read_operations = evidence.merge_read_operations.saturating_add(operations); + Ok(()) +} + +fn open_counted_fixed_reader( + root: &StableDirectory, + name: &str, + evidence: &mut GraphConstructionEvidence, +) -> Result<(BufReader>, IoCounter), GfError> { + let file = root.open_child_file(OsStr::new(name)).map_err(storage)?; + account_sequential_read(file.metadata().map_err(storage)?.len(), evidence); + let counter = IoCounter::default(); + Ok(( + BufReader::with_capacity( + BLOCK_BYTES, + CountingRead { + inner: file, + counter: counter.clone(), + }, + ), + counter, + )) +} + +fn account_fixed_write_operations( + receipt: &ArtifactReceipt, + evidence: &mut GraphConstructionEvidence, +) -> Result<(), GfError> { + if (receipt.bytes == 0) != (receipt.write_operations == 0) { + return Err(storage("fixed-run write bytes and submissions disagree")); + } + evidence.merge_write_operations = evidence + .merge_write_operations + .saturating_add(receipt.write_operations); + Ok(()) +} + fn convert_identity_run( root: &StableDirectory, receipt: &ConstructionChunkReceipt, @@ -3962,7 +4515,14 @@ fn convert_identity_run( .create_replaceable_child_file(OsStr::new(&temporary)) .map_err(storage)?; let identity = file_identity(&file).map_err(storage)?; - let mut reader = BufReader::with_capacity(BLOCK_BYTES, input); + let read_counter = IoCounter::default(); + let mut reader = BufReader::with_capacity( + BLOCK_BYTES, + CountingRead { + inner: input, + counter: read_counter.clone(), + }, + ); let hashing = HashingWriter::new(file); let mut writer = BufWriter::with_capacity(BLOCK_BYTES, hashing); let mut digest = Sha256::new(); @@ -3980,14 +4540,29 @@ fn convert_identity_run( if bytes != receipt.identities.bytes || hex(&digest.finalize()) != receipt.identities.sha256 { return Err(storage("identity source content changed before merge")); } + account_fixed_read_operations(&read_counter, evidence)?; writer.flush().map_err(storage)?; writer.get_ref().inner.sync_all().map_err(storage)?; - account_sequential_write(bytes.saturating_mul(2), evidence); + account_sequential_write(writer.get_ref().bytes, evidence); + let output_receipt = ArtifactReceipt { + name: output.to_owned(), + bytes: writer.get_ref().bytes, + allocated_bytes: graphforge_filesystem::file_space_usage(&writer.get_ref().inner) + .map_err(storage)? + .allocated_bytes, + sha256: hex(&writer.get_ref().digest.clone().finalize()), + identity: identity.into(), + write_operations: writer.get_ref().operations, + fsync_operations: 2, + }; drop(writer); root.install_child(OsStr::new(&temporary), identity, OsStr::new(output)) .map_err(storage)?; root.sync().map_err(storage)?; construction_failpoint("shape.fixed.after_install"); + persist_shape_receipt(root, &output_receipt)?; + record_shape_artifact_install(evidence, &output_receipt)?; + account_fixed_write_operations(&output_receipt, evidence)?; evidence.merge_fsync_operations = evidence.merge_fsync_operations.saturating_add(2); Ok(()) } @@ -4014,7 +4589,14 @@ fn copy_authenticated_run( .create_replaceable_child_file(OsStr::new(&temporary)) .map_err(storage)?; let identity = file_identity(&file).map_err(storage)?; - let mut reader = BufReader::with_capacity(BLOCK_BYTES, input); + let read_counter = IoCounter::default(); + let mut reader = BufReader::with_capacity( + BLOCK_BYTES, + CountingRead { + inner: input, + counter: read_counter.clone(), + }, + ); let hashing = HashingWriter::new(file); let mut writer = BufWriter::with_capacity(BLOCK_BYTES, hashing); let mut digest = Sha256::new(); @@ -4029,9 +4611,14 @@ fn copy_authenticated_run( if bytes != receipt.bytes || hex(&digest.finalize()) != receipt.sha256 { return Err(storage("construction merge source content changed")); } + account_fixed_read_operations(&read_counter, evidence)?; writer.flush().map_err(storage)?; writer.get_ref().inner.sync_all().map_err(storage)?; account_sequential_write(bytes, evidence); + let allocated_bytes = graphforge_filesystem::file_space_usage(&writer.get_ref().inner) + .map_err(storage)? + .allocated_bytes; + let write_operations = writer.get_ref().operations; drop(writer); root.install_child(OsStr::new(&temporary), identity, OsStr::new(output)) .map_err(storage)?; @@ -4039,12 +4626,15 @@ fn copy_authenticated_run( let output_receipt = ArtifactReceipt { name: output.to_owned(), bytes, + allocated_bytes, sha256: receipt.sha256.clone(), identity: identity.into(), - write_operations: bytes.div_ceil(BLOCK_BYTES as u64), + write_operations, fsync_operations: 2, }; persist_shape_receipt(root, &output_receipt)?; + record_shape_artifact_install(evidence, &output_receipt)?; + account_fixed_write_operations(&output_receipt, evidence)?; evidence.merge_fsync_operations = evidence.merge_fsync_operations.saturating_add(2); Ok(()) } @@ -4120,7 +4710,7 @@ impl FixedMergeAccumulator { )?; for input in inputs { if input.starts_with("merge-") { - unlink_named(root, &input)?; + unlink_shape_artifact(root, &input, evidence)?; } } level += 1; @@ -4179,7 +4769,7 @@ impl FixedMergeAccumulator { )?; for input in inputs { if input.starts_with("merge-") { - unlink_named(root, &input)?; + unlink_shape_artifact(root, &input, evidence)?; } } output @@ -4202,6 +4792,7 @@ fn merge_fixed_group( cancelled: &mut impl FnMut() -> bool, evidence: &mut GraphConstructionEvidence, ) -> Result { + let read_counter = IoCounter::default(); let mut readers = inputs .iter() .map(|name| { @@ -4210,7 +4801,13 @@ fn merge_fixed_group( if let Ok(metadata) = file.metadata() { account_sequential_read(metadata.len(), evidence); } - BufReader::with_capacity(BLOCK_BYTES, file) + BufReader::with_capacity( + BLOCK_BYTES, + CountingRead { + inner: file, + counter: read_counter.clone(), + }, + ) }) .map_err(storage) }) @@ -4251,11 +4848,15 @@ fn merge_fixed_group( } } writer.flush().map_err(storage)?; + account_fixed_read_operations(&read_counter, evidence)?; writer.get_ref().inner.sync_all().map_err(storage)?; account_sequential_write(writer.get_ref().bytes, evidence); let receipt = ArtifactReceipt { name: output.to_owned(), bytes: writer.get_ref().bytes, + allocated_bytes: graphforge_filesystem::file_space_usage(&writer.get_ref().inner) + .map_err(storage)? + .allocated_bytes, sha256: hex(&writer.get_ref().digest.clone().finalize()), identity: identity.into(), write_operations: writer.get_ref().operations, @@ -4267,11 +4868,10 @@ fn merge_fixed_group( root.sync().map_err(storage)?; construction_failpoint("shape.fixed_merge.after_install"); persist_shape_receipt(root, &receipt)?; + record_shape_artifact_install(evidence, &receipt)?; + account_fixed_write_operations(&receipt, evidence)?; evidence.merge_fsync_operations = evidence.merge_fsync_operations.saturating_add(2); evidence.merge_groups = evidence.merge_groups.saturating_add(1); - evidence.peak_merge_temporary_bytes = evidence - .peak_merge_temporary_bytes - .max(measured_shape_bytes(root)?); Ok(receipt) } @@ -4451,6 +5051,9 @@ fn merge_row_group( let receipt = ArtifactReceipt { name: output.to_owned(), bytes: hashing.bytes, + allocated_bytes: graphforge_filesystem::file_space_usage(&hashing.inner) + .map_err(storage)? + .allocated_bytes, sha256: hex(&hashing.digest.clone().finalize()), identity: identity.into(), write_operations: hashing.operations, @@ -4462,9 +5065,9 @@ fn merge_row_group( root.sync().map_err(storage)?; construction_failpoint("shape.row_merge.after_install"); persist_shape_receipt(root, &receipt)?; + record_shape_artifact_install(evidence, &receipt)?; evidence.merge_fsync_operations = evidence.merge_fsync_operations.saturating_add(4); evidence.merge_groups = evidence.merge_groups.saturating_add(1); - evidence.merge_written_bytes = evidence.merge_written_bytes.saturating_add(receipt.bytes); evidence.parquet_write_bytes = evidence.parquet_write_bytes.saturating_add(receipt.bytes); evidence.parquet_write_operations = evidence .parquet_write_operations @@ -4472,9 +5075,6 @@ fn merge_row_group( for counter in counters { counter.add_to(evidence); } - evidence.peak_merge_temporary_bytes = evidence - .peak_merge_temporary_bytes - .max(measured_shape_bytes(root)?); Ok(receipt) } @@ -4541,7 +5141,7 @@ impl RowMergeAccumulator { )?; for input in inputs { if input.starts_with("merge-rows-") { - unlink_named(root, &input)?; + unlink_shape_artifact(root, &input, evidence)?; } } level += 1; @@ -4586,7 +5186,7 @@ impl RowMergeAccumulator { )?; for input in inputs { if input.starts_with("merge-rows-") { - unlink_named(root, &input)?; + unlink_shape_artifact(root, &input, evidence)?; } } return Ok(receipt); @@ -4612,7 +5212,7 @@ impl RowMergeAccumulator { )?; for input in inputs { if input.starts_with("merge-rows-") { - unlink_named(root, &input)?; + unlink_shape_artifact(root, &input, evidence)?; } } output_name @@ -4965,15 +5565,8 @@ fn validate_staged_details( cancelled: &mut impl FnMut() -> bool, evidence: &mut GraphConstructionEvidence, ) -> Result<(u64, u64), GfError> { - let mut identities = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(identities_name)) - .map_err(storage)?, - ); - account_sequential_read( - identities.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); + let (mut identities, identities_counter) = + open_counted_fixed_reader(root, identities_name, evidence)?; let mut nodes = 0_u64; let mut edges = 0_u64; while let Some(record) = read_fixed::(&mut identities)? { @@ -4990,6 +5583,7 @@ fn validate_staged_details( reject_cancelled(cancelled)?; } } + account_fixed_read_operations(&identities_counter, evidence)?; let count = |name: Option<&str>, width: u64| -> Result { let Some(name) = name else { return Ok(0) }; let bytes = root @@ -5035,17 +5629,14 @@ fn reject_staged_base_conflicts( cancelled: &mut impl FnMut() -> bool, evidence: &mut GraphConstructionEvidence, ) -> Result<(), GfError> { - let mut reader = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(identities_name)) - .map_err(storage)?, - ); + let (mut reader, reader_counter) = open_counted_fixed_reader(root, identities_name, evidence)?; loop { let mut requested = Vec::with_capacity(window_rows); for _ in 0..window_rows { let Some(record) = read_fixed::(&mut reader)? else { break; }; + account_merge_read::(evidence); requested.push(Uuid::from_bytes( record[..16].try_into().expect("fixed UUID"), )); @@ -5066,6 +5657,7 @@ fn reject_staged_base_conflicts( } reject_cancelled(cancelled)?; } + account_fixed_read_operations(&reader_counter, evidence)?; base.revalidate()?; Ok(()) } @@ -5084,15 +5676,8 @@ fn validate_unified_and_details( cancelled: &mut impl FnMut() -> bool, evidence: &mut GraphConstructionEvidence, ) -> Result<(u64, u64, u64, u64), GfError> { - let mut identities = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(identities_name)) - .map_err(storage)?, - ); - account_sequential_read( - identities.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); + let (mut identities, identities_counter) = + open_counted_fixed_reader(root, identities_name, evidence)?; let mut node_count = 0_u64; let mut edge_count = 0_u64; let mut new_nodes = 0_u64; @@ -5166,6 +5751,7 @@ fn validate_unified_and_details( let max_edge = base_max_edge .checked_add(new_edges) .ok_or_else(|| storage("edge surrogate overflow"))?; + account_fixed_read_operations(&identities_counter, evidence)?; Ok((node_count, edge_count, max_node, max_edge)) } @@ -5180,25 +5766,13 @@ fn validate_detail_domain( let Some(details_name) = details_name else { return Ok(()); }; - let mut identities = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(identities_name)) - .map_err(storage)?, - ); - let mut details = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(details_name)) - .map_err(storage)?, - ); - account_sequential_read( - identities.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); - account_sequential_read( - details.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); + let (mut identities, identities_counter) = + open_counted_fixed_reader(root, identities_name, evidence)?; + let (mut details, details_counter) = open_counted_fixed_reader(root, details_name, evidence)?; let mut identity = read_fixed::(&mut identities)?; + if identity.is_some() { + account_merge_read::(evidence); + } let mut count = 0_u64; while let Some(detail) = read_fixed::(&mut details)? { while identity @@ -5235,6 +5809,8 @@ fn validate_detail_domain( "identity domain contains a row without canonical detail", )); } + account_fixed_read_operations(&identities_counter, evidence)?; + account_fixed_read_operations(&details_counter, evidence)?; Ok(()) } @@ -5256,24 +5832,10 @@ fn validate_endpoints( }; } let endpoints_name = endpoints_name.ok_or_else(|| storage("new edges lack endpoints"))?; - let mut identities = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(identities_name)) - .map_err(storage)?, - ); - let mut endpoints = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(endpoints_name)) - .map_err(storage)?, - ); - account_sequential_read( - identities.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); - account_sequential_read( - endpoints.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); + let (mut identities, identities_counter) = + open_counted_fixed_reader(root, identities_name, evidence)?; + let (mut endpoints, endpoints_counter) = + open_counted_fixed_reader(root, endpoints_name, evidence)?; let mut identity = read_fixed::(&mut identities)?; let mut endpoint_count = 0_u64; while let Some(endpoint) = read_fixed::(&mut endpoints)? { @@ -5304,6 +5866,8 @@ fn validate_endpoints( "edge endpoint cardinality differs from edge domain", )); } + account_fixed_read_operations(&identities_counter, evidence)?; + account_fixed_read_operations(&endpoints_counter, evidence)?; Ok(()) } @@ -5321,15 +5885,7 @@ fn assign_surrogates( .create_replaceable_child_file(OsStr::new(&temporary)) .map_err(storage)?; let identity = file_identity(&file).map_err(storage)?; - let mut reader = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(input_name)) - .map_err(storage)?, - ); - account_sequential_read( - reader.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); + let (mut reader, reader_counter) = open_counted_fixed_reader(root, input_name, evidence)?; let hashing = HashingWriter::new(file); let mut writer = BufWriter::with_capacity(BLOCK_BYTES, hashing); let mut count = 0_u64; @@ -5362,12 +5918,16 @@ fn assign_surrogates( reject_cancelled(cancelled)?; } } + account_fixed_read_operations(&reader_counter, evidence)?; writer.flush().map_err(storage)?; writer.get_ref().inner.sync_all().map_err(storage)?; account_sequential_write(writer.get_ref().bytes, evidence); let output_receipt = ArtifactReceipt { name: output.to_owned(), bytes: writer.get_ref().bytes, + allocated_bytes: graphforge_filesystem::file_space_usage(&writer.get_ref().inner) + .map_err(storage)? + .allocated_bytes, sha256: hex(&writer.get_ref().digest.clone().finalize()), identity: identity.into(), write_operations: writer.get_ref().operations, @@ -5378,6 +5938,8 @@ fn assign_surrogates( .map_err(storage)?; root.sync().map_err(storage)?; persist_shape_receipt(root, &output_receipt)?; + record_shape_artifact_install(evidence, &output_receipt)?; + account_fixed_write_operations(&output_receipt, evidence)?; evidence.merge_fsync_operations = evidence.merge_fsync_operations.saturating_add(2); Ok(output.to_owned()) } @@ -5396,24 +5958,10 @@ fn resolve_endpoint_surrogates( let Some(endpoints_name) = endpoints_name else { return Ok(None); }; - let mut identities = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(identities_name)) - .map_err(storage)?, - ); - let mut endpoints = BufReader::with_capacity( - BLOCK_BYTES, - root.open_child_file(OsStr::new(endpoints_name)) - .map_err(storage)?, - ); - account_sequential_read( - identities.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); - account_sequential_read( - endpoints.get_ref().metadata().map_err(storage)?.len(), - evidence, - ); + let (mut identities, identities_counter) = + open_counted_fixed_reader(root, identities_name, evidence)?; + let (mut endpoints, endpoints_counter) = + open_counted_fixed_reader(root, endpoints_name, evidence)?; let mut identity = read_fixed::(&mut identities)?; let mut window = Vec::<[u8; RESOLVED_ENDPOINT_WIDTH]>::with_capacity(window_rows); let mut resolved = FixedMergeAccumulator::new("merge-resolved", fan_in, false); @@ -5478,8 +6026,10 @@ fn resolve_endpoint_surrogates( window.sort_unstable(); let name = format!("merge-resolved-source-{sequence:020}.run"); let receipt = write_fixed_run(root, &name, &window)?; + record_shape_artifact_install(evidence, &receipt)?; evidence.merge_written_bytes = evidence.merge_written_bytes.saturating_add(receipt.bytes); + account_fixed_write_operations(&receipt, evidence)?; evidence.merge_written_records = evidence .merge_written_records .saturating_add(window.len() as u64); @@ -5500,7 +6050,9 @@ fn resolve_endpoint_surrogates( window.sort_unstable(); let name = format!("merge-resolved-source-{sequence:020}.run"); let receipt = write_fixed_run(root, &name, &window)?; + record_shape_artifact_install(evidence, &receipt)?; evidence.merge_written_bytes = evidence.merge_written_bytes.saturating_add(receipt.bytes); + account_fixed_write_operations(&receipt, evidence)?; evidence.merge_written_records = evidence .merge_written_records .saturating_add(window.len() as u64); @@ -5513,6 +6065,8 @@ fn resolve_endpoint_surrogates( .peak_resolved_endpoint_name_slots .max(resolved.slot_count() as u64); } + account_fixed_read_operations(&identities_counter, evidence)?; + account_fixed_read_operations(&endpoints_counter, evidence)?; resolved.finish_optional::(root, cancelled, evidence) } @@ -6230,6 +6784,19 @@ fn replace_control( Ok(()) } +/// Persist only resumable allocation state. Transition history is live +/// operation evidence; serializing it would make the fixed-size checkpoint +/// grow with every accepted chunk. The exact current union and numeric peak +/// remain durable. +fn replace_checkpoint_control( + root: &StableDirectory, + checkpoint: &Checkpoint, +) -> Result<(), GfError> { + let mut durable = checkpoint.clone(); + durable.evidence.storage_allocation_transitions.clear(); + replace_control(root, CHECKPOINT, &durable) +} + fn control_limit(target: &str) -> u64 { if target == SHAPE_INTENT { MAX_SHAPE_CONTROL_BYTES @@ -7413,6 +7980,27 @@ mod tests { assert!(session.evidence().write_operations < session.evidence().input_rows); assert!(session.evidence().fsync_operations > 0); assert!(session.evidence().peak_accounted_live_bytes > 0); + let staging_storage = + &session.evidence().storage_current[&crate::ArtifactCategory::ConstructionStaging]; + assert_eq!(staging_storage.logical_references, chunks * 3); + assert_eq!(staging_storage.physical_objects, chunks * 3); + assert_eq!( + session.evidence().storage_transient_peak_allocated_bytes + [&crate::ArtifactCategory::ConstructionStaging], + staging_storage.allocated_bytes + ); + assert_eq!( + session + .evidence() + .storage_transient_peak_total_allocated_bytes, + session + .evidence() + .storage_current + .values() + .map(|totals| totals.allocated_bytes) + .sum::() + ); + let persisted_storage = session.evidence().storage_current.clone(); let checkpoint_bytes = session .root .open_child_file(OsStr::new(CHECKPOINT)) @@ -7421,12 +8009,80 @@ mod tests { .unwrap() .len(); assert!(checkpoint_bytes < MAX_CONTROL_BYTES); + drop(session); + let mut session = GraphConstructionSession::resume_with_mode_and_lifecycle( + root.path(), + Uuid::from_u128(operation), + graphforge_core::OntologyMode::Exploratory, + GraphConstructionBudgets::default(), + crate::filesystem_admission::ProjectLifecycleMode::Durable, + ) + .unwrap(); + assert_eq!(session.evidence().storage_current, persisted_storage); session.seal().unwrap(); assert_eq!(session.state(), GraphConstructionState::Sealed); assert!(session.evidence().authentication_read_bytes > 0); } } + #[test] + fn checkpoint_compacts_live_transition_history_without_losing_peak_or_union() { + let root = TempDir::new().unwrap(); + let operation = 9_901_u128; + let mut session = open(&root, operation); + session + .append(ConstructionChunkKind::Node, "n", &node_batch(1, 32)) + .unwrap(); + let active = session + .checkpoint + .evidence + .storage_active_identity_allocated_bytes + .clone(); + let peak = session + .checkpoint + .evidence + .storage_transient_peak_total_allocated_bytes; + let transition = session + .checkpoint + .evidence + .storage_allocation_transitions + .last() + .unwrap() + .clone(); + session.checkpoint.evidence.storage_allocation_transitions = vec![transition; 20_000]; + replace_checkpoint_control(&session.root, &session.checkpoint).unwrap(); + assert!( + session + .root + .open_child_file(OsStr::new(CHECKPOINT)) + .unwrap() + .metadata() + .unwrap() + .len() + < MAX_CONTROL_BYTES + ); + drop(session); + let reopened = GraphConstructionSession::resume_with_mode_and_lifecycle( + root.path(), + Uuid::from_u128(operation), + graphforge_core::OntologyMode::Exploratory, + GraphConstructionBudgets::default(), + crate::filesystem_admission::ProjectLifecycleMode::Durable, + ) + .unwrap(); + assert_eq!( + reopened.evidence().storage_active_identity_allocated_bytes, + active + ); + assert_eq!( + reopened + .evidence() + .storage_transient_peak_total_allocated_bytes, + peak + ); + assert_eq!(reopened.evidence().storage_allocation_transitions.len(), 1); + } + #[test] fn million_chunk_online_scheduler_has_logarithmic_name_state() { let slots = online_merge_name_slot_bound(1_000_000, 32); @@ -7649,6 +8305,8 @@ mod tests { assert!(session.evidence().merge_read_blocks > 0); assert!(session.evidence().merge_write_blocks > 0); assert!(session.evidence().merge_fsync_operations > 0); + assert!(session.evidence().merge_read_operations > 0); + assert!(session.evidence().merge_write_operations > 0); assert!(session.evidence().parquet_read_operations > 0); assert!(session.evidence().parquet_write_operations > 0); if chunks == 4 { @@ -8549,6 +9207,16 @@ mod tests { ) .unwrap(); assert_eq!(resumed.accepted_chunks(), accepted, "{failpoint}"); + if accepted == 1 { + assert!( + resumed.evidence().recovery_application_read_bytes > 0, + "accepted interrupted append must report recovery bytes: {failpoint}" + ); + assert!( + resumed.evidence().recovery_application_read_operations > 0, + "accepted interrupted append must report recovery calls: {failpoint}" + ); + } if accepted == 0 { resumed .append(ConstructionChunkKind::Node, "nodes", &node_batch(1, 8)) @@ -8560,6 +9228,13 @@ mod tests { #[test] fn shape_inventory_and_evidence_commit_recover_without_double_counting() { + fn without_native_identities( + mut evidence: GraphConstructionEvidence, + ) -> GraphConstructionEvidence { + evidence.storage_active_identity_allocated_bytes.clear(); + evidence.storage_allocation_transitions.clear(); + evidence + } let reference_root = TempDir::new().unwrap(); let mut reference = GraphConstructionSession::open( reference_root.path(), @@ -8578,7 +9253,7 @@ mod tests { reference .shape_canonical_with_cancellation(|| false) .unwrap(); - let expected = reference.evidence().clone(); + let expected = without_native_identities(reference.evidence().clone()); for failpoint in [ "shape.fixed.after_install", @@ -8610,7 +9285,11 @@ mod tests { ) .unwrap(); resumed.shape_canonical_with_cancellation(|| false).unwrap(); - assert_eq!(resumed.evidence(), &expected, "{failpoint}"); + assert_eq!( + without_native_identities(resumed.evidence().clone()), + expected, + "{failpoint}" + ); } } diff --git a/crates/graphforge-storage/src/graph_construction_encoding.rs b/crates/graphforge-storage/src/graph_construction_encoding.rs index 3a41a9b64..edd80a9f0 100644 --- a/crates/graphforge-storage/src/graph_construction_encoding.rs +++ b/crates/graphforge-storage/src/graph_construction_encoding.rs @@ -1928,23 +1928,41 @@ fn authenticate_inventory( pub(crate) fn authenticate_inventory_control_for_publication( source: &StableDirectory, inventory: &GraphConstructionEncoding, -) -> Result { +) -> Result { let encoded = source .open_child_directory(OsStr::new(ENCODED_ROOT)) .map_err(storage)?; - let recorded = read_inventory(&encoded)? - .ok_or_else(|| storage("canonical encoding inventory is absent"))?; + let file = encoded + .open_child_file(OsStr::new(INVENTORY)) + .map_err(storage)?; + if file.metadata().map_err(storage)?.len() > MAX_INVENTORY_BYTES { + return Err(storage("canonical inventory exceeds bound")); + } + let counter = IoCounter::default(); + let recorded: GraphConstructionEncoding = serde_json::from_reader(BufReader::with_capacity( + COPY_BUFFER_BYTES, + CountingInput { + inner: file, + counter: counter.clone(), + }, + )) + .map_err(storage)?; if &recorded != inventory { return Err(storage( "publication inventory differs from durable encoding", )); } - encoded - .open_child_file(OsStr::new(INVENTORY)) - .map_err(storage)? - .metadata() - .map(|metadata| metadata.len()) - .map_err(storage) + let (read_bytes, read_calls) = counter.values(); + Ok(PublicationControlIoEvidence { + read_bytes, + read_calls, + }) +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct PublicationControlIoEvidence { + pub(crate) read_bytes: u64, + pub(crate) read_calls: u64, } fn install_json( diff --git a/crates/graphforge-storage/src/graph_files.rs b/crates/graphforge-storage/src/graph_files.rs index 2f8c29b9d..f271f2f80 100644 --- a/crates/graphforge-storage/src/graph_files.rs +++ b/crates/graphforge-storage/src/graph_files.rs @@ -115,6 +115,16 @@ pub struct GraphFilesOpenEvidence { pub files_reused: u64, /// Logical bytes represented by reused immutable objects. pub bytes_reused: u64, + /// Bytes returned by application-level hydration/verification reads. + pub application_read_bytes: u64, + /// Non-empty application-level hydration/verification reads. + pub application_read_calls: u64, + /// Bytes submitted by application-level copy operations. + pub application_write_bytes: u64, + /// Application-level copy submissions. + pub application_write_calls: u64, + /// File and directory durability barriers completed while hydrating. + pub fsync_calls: u64, } /// Open/materialization strategy for a file-backed graph. @@ -331,13 +341,26 @@ pub fn stage_graph_tree( fs::create_dir_all(parent) .map_err(|error| storage("create graph tree directory", parent, error))?; } - let digest = copy_regular_file(&source, &destination)?; + let copied = copy_regular_file(&source, &destination)?; + let digest = copied.digest; + evidence.application_read_bytes = evidence + .application_read_bytes + .saturating_add(copied.read_bytes); + evidence.application_read_calls = evidence + .application_read_calls + .saturating_add(copied.read_calls); + evidence.application_write_bytes = evidence + .application_write_bytes + .saturating_add(copied.write_bytes); + evidence.application_write_calls = evidence + .application_write_calls + .saturating_add(copied.write_calls); + evidence.fsync_calls = evidence.fsync_calls.saturating_add(copied.fsync_calls); if hex_digest(digest) != entry.content_sha256 { return Err(validation( "graph tree source digest does not match inventory", )); } - sync_file(&destination)?; evidence.files_validated = evidence.files_validated.saturating_add(1); evidence.bytes_validated = evidence.bytes_validated.saturating_add(entry.byte_length); evidence.files_copied = evidence.files_copied.saturating_add(1); @@ -352,7 +375,9 @@ pub fn stage_graph_tree( false, )?; } - sync_directory_tree(&destination_root)?; + evidence.fsync_calls = evidence + .fsync_calls + .saturating_add(sync_directory_tree(&destination_root)?); verify_graph_tree(&destination_root, inventory)?; Ok(evidence) } @@ -446,8 +471,22 @@ pub fn materialize_graph_tree( fs::create_dir_all(parent) .map_err(|error| storage("create private graph directory", parent, error))?; } - copy_regular_file(&source, &destination)?; + let copied = copy_regular_file(&source, &destination)?; + evidence.application_read_bytes = evidence + .application_read_bytes + .saturating_add(copied.read_bytes); + evidence.application_read_calls = evidence + .application_read_calls + .saturating_add(copied.read_calls); + evidence.application_write_bytes = evidence + .application_write_bytes + .saturating_add(copied.write_bytes); + evidence.application_write_calls = evidence + .application_write_calls + .saturating_add(copied.write_calls); + evidence.fsync_calls = evidence.fsync_calls.saturating_add(copied.fsync_calls); make_private_copy_owner_writable(&destination)?; + evidence.fsync_calls = evidence.fsync_calls.saturating_add(1); evidence.files_copied = evidence.files_copied.saturating_add(1); evidence.bytes_copied = evidence.bytes_copied.saturating_add(entry.byte_length); } @@ -466,6 +505,7 @@ pub fn pinned_open_evidence(inventory: &GraphFilesInventory) -> GraphFilesOpenEv files_opened_in_place: u64::try_from(inventory.files.len()).unwrap_or(u64::MAX), files_reused: 0, bytes_reused: 0, + ..GraphFilesOpenEvidence::default() } } @@ -788,16 +828,52 @@ fn ensure_empty_directory(target: &Path) -> Result<(), GfError> { Ok(()) } -fn copy_regular_file(source: &Path, destination: &Path) -> Result<[u8; 32], GfError> { +struct CopyIoEvidence { + digest: [u8; 32], + read_bytes: u64, + read_calls: u64, + write_bytes: u64, + write_calls: u64, + fsync_calls: u64, +} + +fn copy_regular_file(source: &Path, destination: &Path) -> Result { reject_link(source)?; // Prefer filesystem copy so sparse/holey sources stay sparse when the OS // supports it (Linux copy_file_range). Digest the destination so staged // bytes remain verified without assembling them into one buffer. - fs::copy(source, destination) + let copied = fs::copy(source, destination) .map_err(|error| storage("copy graph source file", destination, error))?; - let digest = hash_file(destination)?; + let (digest, read_bytes, read_calls) = hash_file_io_counted(destination)?; sync_file(destination)?; - Ok(digest) + Ok(CopyIoEvidence { + digest, + read_bytes, + read_calls, + write_bytes: copied, + write_calls: u64::from(copied != 0), + fsync_calls: 1, + }) +} + +fn hash_file_io_counted(path: &Path) -> Result<([u8; 32], u64, u64), GfError> { + let mut file = File::open(path).map_err(|error| storage("open graph file", path, error))?; + let mut digest = Sha256::new(); + let mut bytes = 0_u64; + let mut calls = 0_u64; + let mut buffer = vec![0_u8; HASH_BUFFER_BYTES]; + loop { + let read = file + .read(&mut buffer) + .map_err(|error| storage("read graph file", path, error))?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + bytes = bytes.saturating_add(read as u64); + calls = calls.saturating_add(1); + } + Ok((digest.finalize().into(), bytes, calls)) } #[cfg(unix)] @@ -824,6 +900,7 @@ fn make_private_copy_owner_writable(path: &Path) -> Result<(), GfError> { sync_file(path) } +#[cfg(test)] fn hash_file(path: &Path) -> Result<[u8; 32], GfError> { hash_file_counted(path).map(|(digest, _)| digest) } @@ -859,7 +936,7 @@ fn sync_file(path: &Path) -> Result<(), GfError> { .map_err(|error| storage("fsync graph file", path, error)) } -fn sync_directory_tree(root: &Path) -> Result<(), GfError> { +fn sync_directory_tree(root: &Path) -> Result { let mut directories = vec![root.to_path_buf()]; let mut index = 0; while index < directories.len() { @@ -879,10 +956,11 @@ fn sync_directory_tree(root: &Path) -> Result<(), GfError> { } index += 1; } + let count = u64::try_from(directories.len()).unwrap_or(u64::MAX); for directory in directories.into_iter().rev() { sync_directory(&directory)?; } - Ok(()) + Ok(count) } fn sync_directory(path: &Path) -> Result<(), GfError> { @@ -1513,6 +1591,14 @@ mod tests { let evidence = stage_graph_tree(source.path(), generation.path(), &inventory).unwrap(); assert_eq!(evidence.files_copied, 2); assert_eq!(evidence.bytes_copied, inventory.total_byte_length); + assert_eq!(evidence.application_read_bytes, inventory.total_byte_length); + assert_eq!(evidence.application_read_calls, 2); + assert_eq!( + evidence.application_write_bytes, + inventory.total_byte_length + ); + assert_eq!(evidence.application_write_calls, 2); + assert_eq!(evidence.fsync_calls, 4); let sealed_source = graph_tree_root(generation.path()).join("properties/Person.parquet"); let mut sealed_permissions = fs::metadata(&sealed_source).unwrap().permissions(); @@ -1528,6 +1614,11 @@ mod tests { .unwrap(); assert_eq!(opened.strategy, GraphFilesOpenStrategy::PrivateMaterialize); assert_eq!(opened.files_copied, 2); + assert_eq!(opened.application_read_bytes, inventory.total_byte_length); + assert_eq!(opened.application_read_calls, 2); + assert_eq!(opened.application_write_bytes, inventory.total_byte_length); + assert_eq!(opened.application_write_calls, 2); + assert_eq!(opened.fsync_calls, 4); let private_copy = private.path().join("properties/Person.parquet"); assert!( fs::metadata(&sealed_source) diff --git a/crates/graphforge-storage/src/graph_object_store.rs b/crates/graphforge-storage/src/graph_object_store.rs index 47c1f1f7d..056e76626 100644 --- a/crates/graphforge-storage/src/graph_object_store.rs +++ b/crates/graphforge-storage/src/graph_object_store.rs @@ -563,6 +563,14 @@ pub struct GraphObjectInstallEvidence { pub bytes_installed: u64, /// Whether an already installed exact object satisfied the request. pub reused_existing: bool, + /// Non-empty source or authentication reads completed by the application. + pub read_calls: u64, + /// Temporary-object write submissions completed by the application. + pub write_calls: u64, + /// Payload bytes submitted to temporary-object writers. + pub write_bytes: u64, + /// File and directory durability barriers completed by this installation. + pub fsync_calls: u64, } /// One-time v1 expanded-tree to v2 object-store migration evidence. @@ -587,6 +595,14 @@ pub struct GraphFilesAppendEvidence { pub payload_bytes_hashed: u64, /// Logical object bytes newly installed. pub bytes_installed: u64, + /// Actual non-empty payload reads performed by object installation. + pub read_calls: u64, + /// Actual payload write submissions performed by object installation. + pub write_calls: u64, + /// Payload bytes submitted to object writers. + pub write_bytes: u64, + /// File and directory durability barriers completed by object installation. + pub fsync_calls: u64, } /// A graph file whose content identity was established by an upstream durable @@ -666,6 +682,100 @@ pub struct GraphObjectGcEvidence { pub objects_removed: u64, /// Physical unreachable bytes removed. pub bytes_removed: u64, + /// Exact native identities and allocated bytes removed by this GC receipt. + pub removed_identity_allocated_bytes: BTreeMap, +} + +/// Capture every sealed CAS object and its lifecycle control by native identity. +/// +/// This is a storage-owned, bounded phase-boundary inventory. It is never used +/// during active ingest; qualification calls it only while holding the CAS +/// shared lifecycle lock, so installed-but-unreferenced objects remain charged +/// until an explicit GC receipt removes them. +pub(crate) fn capture_retained_graph_object_identities( + root: &Path, +) -> Result, GfError> { + const MAX_RETAINED_OBJECTS: usize = 4_000_000; + if !root.join(GRAPH_OBJECTS_DIR).exists() { + return Ok(BTreeMap::new()); + } + let cas = ReadOnlyCasRoot::open(root)?; + let mut identities = BTreeMap::new(); + add_retained_identity(&mut identities, &cas.lifecycle)?; + let prefixes = cas + .sha256 + .child_names_bounded(256) + .map_err(|error| storage("inventory stable graph object prefixes", root, error))?; + let mut remaining = MAX_RETAINED_OBJECTS; + for prefix in prefixes { + let prefix_text = prefix + .to_str() + .ok_or_else(|| validation("graph object prefix is not UTF-8"))?; + if prefix_text.len() != 2 + || !prefix_text + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err(validation("graph object prefix is not canonical")); + } + let bucket = cas + .sha256 + .open_child_directory(&prefix) + .map_err(|error| storage("open retained graph object bucket", root, error))?; + let objects = bucket + .child_names_bounded(remaining) + .map_err(|error| storage("inventory retained graph object bucket", root, error))?; + remaining = remaining.checked_sub(objects.len()).ok_or_else(|| { + validation("retained graph object inventory exceeds attribution bound") + })?; + for object in objects { + let suffix = object + .to_str() + .ok_or_else(|| validation("graph object name is not UTF-8"))?; + validate_digest(&format!("{prefix_text}{suffix}"))?; + let file = bucket + .open_child_file(&object) + .map_err(|error| storage("open retained graph object", root, error))?; + add_retained_identity(&mut identities, &file)?; + } + } + Ok(identities) +} + +fn add_retained_identity( + identities: &mut BTreeMap, + file: &File, +) -> Result<(), GfError> { + let identity = graphforge_filesystem::file_identity(file).map_err(|error| { + storage( + "identify retained graph object", + Path::new(GRAPH_OBJECTS_DIR), + error, + ) + })?; + let usage = graphforge_filesystem::file_space_usage(file).map_err(|error| { + storage( + "measure retained graph object", + Path::new(GRAPH_OBJECTS_DIR), + error, + ) + })?; + let key = retained_identity_key(identity); + match identities.insert(key, usage.allocated_bytes) { + Some(existing) if existing != usage.allocated_bytes => { + Err(validation("retained graph object allocation changed")) + } + _ => Ok(()), + } +} + +fn retained_identity_key(identity: graphforge_filesystem::FileIdentity) -> String { + use std::fmt::Write as _; + let mut key = format!("{:016x}:", identity.volume_serial); + for byte in identity.file_id { + write!(&mut key, "{byte:02x}").expect("writing to String cannot fail"); + } + key } /// Trace compact generation roots, then sweep unreachable CAS objects. @@ -855,7 +965,21 @@ pub(crate) fn gc_graph_objects_guarded( error, ) })?; - candidates.push((prefix.clone(), object, identity, metadata.len())); + let allocation = + graphforge_filesystem::file_space_usage(&file).map_err(|error| { + storage( + "measure graph object candidate", + &guard.cas.diagnostic_root, + error, + ) + })?; + candidates.push(( + prefix.clone(), + object, + identity, + metadata.len(), + allocation.allocated_bytes, + )); } } } @@ -863,7 +987,7 @@ pub(crate) fn gc_graph_objects_guarded( objects_marked: u64::try_from(marked.len()).unwrap_or(u64::MAX), ..GraphObjectGcEvidence::default() }; - for (prefix, object, identity, bytes) in candidates { + for (prefix, object, identity, bytes, allocated) in candidates { let bucket = guard .cas .sha256 @@ -884,8 +1008,18 @@ pub(crate) fn gc_graph_objects_guarded( error, ) })?; + bucket.sync().map_err(|error| { + storage( + "sync graph object bucket after GC removal", + &guard.cas.diagnostic_root, + error, + ) + })?; evidence.objects_removed = evidence.objects_removed.saturating_add(1); evidence.bytes_removed = evidence.bytes_removed.saturating_add(bytes); + evidence + .removed_identity_allocated_bytes + .insert(retained_identity_key(identity), allocated); } Ok(evidence) } @@ -1001,6 +1135,10 @@ fn append_graph_files_v2_inner( evidence.bytes_installed = evidence .bytes_installed .saturating_add(installed.bytes_installed); + evidence.read_calls = evidence.read_calls.saturating_add(installed.read_calls); + evidence.write_calls = evidence.write_calls.saturating_add(installed.write_calls); + evidence.write_bytes = evidence.write_bytes.saturating_add(installed.write_bytes); + evidence.fsync_calls = evidence.fsync_calls.saturating_add(installed.fsync_calls); let relative_path = relative .to_str() .ok_or_else(|| validation("sealed graph path is not UTF-8"))? @@ -1453,7 +1591,14 @@ fn install_graph_object_bytes_with_lease( // file verification below is an application-observed payload read. Ok(0) }) - .map(|evidence| (digest, evidence)) + .map(|mut evidence| { + if !evidence.reused_existing { + evidence.write_bytes = expected_length; + evidence.write_calls = u64::from(!bytes.is_empty()); + evidence.fsync_calls = evidence.fsync_calls.saturating_add(1); + } + (digest, evidence) + }) } /// Stream, hash, and install a new payload object from a regular source file. @@ -1482,7 +1627,9 @@ fn install_graph_object_file_with_lease( "graph object source is not the declared regular file", )); } - install_object( + let read_calls = std::cell::Cell::new(0_u64); + let write_calls = std::cell::Cell::new(0_u64); + let result = install_object( &lease.cas, expected_digest, expected_length, @@ -1500,6 +1647,7 @@ fn install_graph_object_file_with_lease( if read == 0 { break; } + read_calls.set(read_calls.get().saturating_add(1)); output.write_all(&buffer[..read]).map_err(|error| { storage( "write temporary graph object", @@ -1507,6 +1655,7 @@ fn install_graph_object_file_with_lease( error, ) })?; + write_calls.set(write_calls.get().saturating_add(1)); hasher.update(&buffer[..read]); total = total.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); } @@ -1524,7 +1673,16 @@ fn install_graph_object_file_with_lease( })?; Ok(total) }, - ) + ); + result.map(|mut evidence| { + if !evidence.reused_existing { + evidence.read_calls = evidence.read_calls.saturating_add(read_calls.get()); + evidence.write_calls = evidence.write_calls.saturating_add(write_calls.get()); + evidence.write_bytes = evidence.write_bytes.saturating_add(expected_length); + evidence.fsync_calls = evidence.fsync_calls.saturating_add(1); + } + evidence + }) } /// Read and cryptographically verify an immutable object. @@ -1582,7 +1740,23 @@ pub fn materialize_graph_objects( ..GraphFilesOpenEvidence::default() }; for entry in &inventory.files { - if materialize_from_cas(&lease.cas, &target_directory, entry)? { + let materialized = materialize_from_cas(&lease.cas, &target_directory, entry)?; + evidence.application_read_bytes = evidence + .application_read_bytes + .saturating_add(materialized.read_bytes); + evidence.application_read_calls = evidence + .application_read_calls + .saturating_add(materialized.read_calls); + evidence.application_write_bytes = evidence + .application_write_bytes + .saturating_add(materialized.write_bytes); + evidence.application_write_calls = evidence + .application_write_calls + .saturating_add(materialized.write_calls); + evidence.fsync_calls = evidence + .fsync_calls + .saturating_add(materialized.fsync_calls); + if materialized.copied { evidence.files_copied = evidence.files_copied.saturating_add(1); evidence.bytes_copied = evidence.bytes_copied.saturating_add(entry.byte_length); } else { @@ -1598,7 +1772,7 @@ fn materialize_from_cas( cas: &CasRoot, target: &StableDirectory, entry: &crate::GraphFileEntry, -) -> Result { +) -> Result { validate_logical_path(Path::new(&entry.relative_path))?; let bucket = cas.digest_bucket(&entry.content_sha256, false)?; let source_name = std::ffi::OsStr::new(&entry.content_sha256[2..]); @@ -1631,8 +1805,7 @@ fn materialize_from_cas( } else { let parent = parent.as_ref().unwrap_or(target); if requires_single_link_materialization(&entry.relative_path) { - copy_single_link_materialized_object(cas, &source, parent, name, entry)?; - return Ok(true); + return copy_single_link_materialized_object(cas, &source, parent, name, entry); } let (installed, installed_identity) = bucket .link_child_into(source_name, &source, source_identity, parent, name) @@ -1643,18 +1816,38 @@ fn materialize_from_cas( error, ) })?; - if let Err(error) = verify_file( + let verified = verify_file_counted( installed, &entry.content_sha256, entry.byte_length, &cas.diagnostic_root, - ) { - let _ = parent.unlink_child_if_identity(name, installed_identity); - return Err(error); + ); + match verified { + Ok(io) => { + return Ok(MaterializeIoEvidence { + read_bytes: io.bytes, + read_calls: io.calls, + ..MaterializeIoEvidence::default() + }); + } + Err(error) => { + let _ = parent.unlink_child_if_identity(name, installed_identity); + return Err(error); + } } } } - Ok(false) + Err(validation("materialization path has no final component")) +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct MaterializeIoEvidence { + copied: bool, + read_bytes: u64, + read_calls: u64, + write_bytes: u64, + write_calls: u64, + fsync_calls: u64, } fn requires_single_link_materialization(relative_path: &str) -> bool { @@ -1675,7 +1868,7 @@ fn copy_single_link_materialized_object( parent: &StableDirectory, name: &std::ffi::OsStr, entry: &crate::GraphFileEntry, -) -> Result<(), GfError> { +) -> Result { let temporary_name = std::ffi::OsString::from(format!( ".ordinal-v4-materialize-{}.tmp", Uuid::new_v4().simple() @@ -1703,8 +1896,8 @@ fn copy_single_link_materialized_object( ) })?; let mut installed = false; - let result = (|| -> Result<(), GfError> { - copy_and_authenticate_materialized_object( + let result = (|| -> Result { + let mut io = copy_and_authenticate_materialized_object( &mut input, &mut output, entry, @@ -1728,6 +1921,7 @@ fn copy_single_link_materialized_object( error, ) })?; + io.fsync_calls = io.fsync_calls.saturating_add(1); let installed = parent.open_child_file(name).map_err(|error| { storage( "open private materialization file", @@ -1747,12 +1941,16 @@ fn copy_single_link_materialized_object( "private materialization file is multiply linked", )); } - verify_file( + let verified = verify_file_counted( installed, &entry.content_sha256, entry.byte_length, &cas.diagnostic_root, - ) + )?; + io.read_bytes = io.read_bytes.saturating_add(verified.bytes); + io.read_calls = io.read_calls.saturating_add(verified.calls); + io.copied = true; + Ok(io) })(); if result.is_err() { let cleanup_name = if installed { name } else { &temporary_name }; @@ -1767,9 +1965,11 @@ fn copy_and_authenticate_materialized_object( output: &mut File, entry: &crate::GraphFileEntry, diagnostic_root: &Path, -) -> Result<(), GfError> { +) -> Result { let mut digest = Sha256::new(); let mut length = 0_u64; + let mut read_calls = 0_u64; + let mut write_calls = 0_u64; let mut buffer = vec![0_u8; BUFFER_BYTES].into_boxed_slice(); loop { let read = input @@ -1778,9 +1978,11 @@ fn copy_and_authenticate_materialized_object( if read == 0 { break; } + read_calls = read_calls.saturating_add(1); output.write_all(&buffer[..read]).map_err(|error| { storage("write private materialization file", diagnostic_root, error) })?; + write_calls = write_calls.saturating_add(1); digest.update(&buffer[..read]); length = length .checked_add(read as u64) @@ -1793,7 +1995,15 @@ fn copy_and_authenticate_materialized_object( } output .sync_all() - .map_err(|error| storage("sync private materialization file", diagnostic_root, error)) + .map_err(|error| storage("sync private materialization file", diagnostic_root, error))?; + Ok(MaterializeIoEvidence { + copied: true, + read_bytes: length, + read_calls, + write_bytes: length, + write_calls, + fsync_calls: 1, + }) } #[cfg(unix)] @@ -2447,24 +2657,23 @@ where #[cfg(windows)] let temporary_identity = temporary.identity(); let bytes_hashed = write_temporary(&mut temporary)?; - let preseal_bytes_hashed = if writer_authenticated || cfg!(windows) { - 0 + let preseal_io = if writer_authenticated || cfg!(windows) { + ReadIoEvidence::default() } else { temporary.rewind().map_err(|error| { storage("rewind temporary graph object", &cas.diagnostic_root, error) })?; - verify_stream( + verify_stream_counted( &mut temporary, digest, expected_length, &cas.diagnostic_root, - )?; - expected_length + )? }; // Windows must close the writable handle and reopen an exact-identity, // protected read handle before publication. That transition authenticates // the complete payload below, so a second pre-seal read would be redundant. - let (installed, sealed_bytes_hashed) = finalize_temporary_object( + let (installed, sealed_bytes_hashed, concurrent_io) = finalize_temporary_object( cas, &bucket, TemporaryObject { @@ -2477,18 +2686,26 @@ where )?; Ok(GraphObjectInstallEvidence { bytes_hashed: bytes_hashed - .saturating_add(preseal_bytes_hashed) + .saturating_add(preseal_io.bytes) .saturating_add(sealed_bytes_hashed) .saturating_add(if installed { 0 } else { expected_length }), bytes_installed: if installed { expected_length } else { 0 }, reused_existing: !installed, + read_calls: preseal_io.calls.saturating_add(concurrent_io.calls), + // The source-copy/authentication submissions are added by the caller. + // Fresh installation always durably synchronizes the destination bucket + // and temporary namespace; reuse performs neither operation here. + fsync_calls: if installed { 2 } else { 0 }, + ..GraphObjectInstallEvidence::default() }) } -fn reused_object_evidence(expected_length: u64) -> GraphObjectInstallEvidence { +fn reused_object_evidence(expected_length: u64, io: ReadIoEvidence) -> GraphObjectInstallEvidence { + debug_assert_eq!(io.bytes, expected_length); GraphObjectInstallEvidence { - bytes_hashed: expected_length, + bytes_hashed: io.bytes, reused_existing: true, + read_calls: io.calls, ..GraphObjectInstallEvidence::default() } } @@ -2512,14 +2729,14 @@ fn try_reuse_existing_object( )); } }; - verify_and_seal_graph_object( + let io = verify_and_seal_graph_object_counted( &file, digest, expected_length, &graph_object_path(&cas.diagnostic_root, digest)?, &cas.diagnostic_root, )?; - Ok(Some(reused_object_evidence(expected_length))) + Ok(Some(reused_object_evidence(expected_length, io))) } #[cfg(windows)] @@ -2530,6 +2747,7 @@ fn try_reuse_existing_object( digest: &str, expected_length: u64, ) -> Result, GfError> { + let mut adoption_io = ReadIoEvidence::default(); let file = match bucket.open_cas_child_file(destination_name) { Ok(file) => file.into_file(), Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), @@ -2547,7 +2765,8 @@ fn try_reuse_existing_object( }, ) })?; - verify_stream(&mut legacy, digest, expected_length, &cas.diagnostic_root)?; + adoption_io = + verify_stream_counted(&mut legacy, digest, expected_length, &cas.diagnostic_root)?; bucket .adopt_legacy_cas_child(destination_name, legacy) .map(graphforge_filesystem::WindowsSealedCasFile::into_file) @@ -2560,14 +2779,17 @@ fn try_reuse_existing_object( })? } }; - verify_and_seal_graph_object( + let io = verify_and_seal_graph_object_counted( &file, digest, expected_length, &graph_object_path(&cas.diagnostic_root, digest)?, &cas.diagnostic_root, )?; - Ok(Some(reused_object_evidence(expected_length))) + let mut evidence = reused_object_evidence(expected_length, io); + evidence.bytes_hashed = adoption_io.bytes.saturating_add(io.bytes); + evidence.read_calls = adoption_io.calls.saturating_add(io.calls); + Ok(Some(evidence)) } fn finalize_temporary_object( @@ -2576,7 +2798,7 @@ fn finalize_temporary_object( temporary: TemporaryObject, digest: &str, expected_length: u64, -) -> Result<(bool, u64), GfError> { +) -> Result<(bool, u64, ReadIoEvidence), GfError> { let destination_name = std::ffi::OsStr::new(&digest[2..]); let temporary_path = cas .diagnostic_root @@ -2584,40 +2806,29 @@ fn finalize_temporary_object( .join(TEMP_DIR) .join(&temporary.name); #[cfg(unix)] - let temporary = { + let (temporary, sealed_io) = { seal_graph_object(&temporary.file, &temporary_path, &cas.diagnostic_root)?; - SealedTemporaryObject { - name: temporary.name, - file: temporary.file, - identity: temporary.identity, - } + ( + SealedTemporaryObject { + name: temporary.name, + file: temporary.file, + identity: temporary.identity, + }, + ReadIoEvidence::default(), + ) }; #[cfg(windows)] - let temporary = transition_temporary_to_sealed_reader( + let (temporary, sealed_io) = transition_temporary_to_sealed_reader( &cas.tmp, temporary, digest, expected_length, &cas.diagnostic_root, )?; - #[cfg(unix)] - let sealed_bytes_hashed = 0; - #[cfg(windows)] - let sealed_bytes_hashed = expected_length; - let sealed_metadata = temporary - .file - .metadata() - .map_err(|error| storage("reinspect fresh graph object", &cas.diagnostic_root, error))?; - if graphforge_filesystem::file_identity(&temporary.file) - .map_err(|error| storage("reidentify fresh graph object", &cas.diagnostic_root, error))? - != temporary.identity - || !sealed_metadata.is_file() - || sealed_metadata.len() != expected_length - || !sealed_metadata.permissions().readonly() - { - return Err(validation("fresh graph object post-hash authority changed")); - } + let sealed_bytes_hashed = sealed_io.bytes; + validate_sealed_temporary(&temporary, expected_length, &cas.diagnostic_root)?; returned_error_boundary("install:temp-sealed")?; + let mut concurrent_io = ReadIoEvidence::default(); let installed = if let Ok((_installed, _identity)) = cas.tmp.link_child_into( &temporary.name, &temporary.file, @@ -2640,7 +2851,7 @@ fn finalize_temporary_object( error, ) })?; - verify_and_seal_graph_object( + concurrent_io = verify_and_seal_graph_object_counted( &existing, digest, expected_length, @@ -2683,7 +2894,35 @@ fn finalize_temporary_object( error, ) })?; - Ok((installed, sealed_bytes_hashed)) + Ok(( + installed, + sealed_bytes_hashed, + ReadIoEvidence { + bytes: sealed_io.bytes.saturating_add(concurrent_io.bytes), + calls: sealed_io.calls.saturating_add(concurrent_io.calls), + }, + )) +} + +fn validate_sealed_temporary( + temporary: &SealedTemporaryObject, + expected_length: u64, + diagnostic_root: &Path, +) -> Result<(), GfError> { + let metadata = temporary + .file + .metadata() + .map_err(|error| storage("reinspect fresh graph object", diagnostic_root, error))?; + let identity = graphforge_filesystem::file_identity(&temporary.file) + .map_err(|error| storage("reidentify fresh graph object", diagnostic_root, error))?; + if identity != temporary.identity + || !metadata.is_file() + || metadata.len() != expected_length + || !metadata.permissions().readonly() + { + return Err(validation("fresh graph object post-hash authority changed")); + } + Ok(()) } #[cfg(windows)] @@ -2693,7 +2932,7 @@ fn transition_temporary_to_sealed_reader( digest: &str, expected_length: u64, diagnostic: &Path, -) -> Result { +) -> Result<(SealedTemporaryObject, ReadIoEvidence), GfError> { temporary.file.sync_all().map_err(|error| { storage( "sync temporary graph object before sealing", @@ -2719,7 +2958,7 @@ fn transition_temporary_to_sealed_reader( "temporary graph object identity changed while sealing", )); } - verify_file( + let io = verify_file_counted( file.try_clone().map_err(|error| { storage( "clone sealed temporary graph object for authentication", @@ -2731,19 +2970,37 @@ fn transition_temporary_to_sealed_reader( expected_length, diagnostic, )?; - Ok(SealedTemporaryObject { - name, - file, - identity, - }) + Ok(( + SealedTemporaryObject { + name, + file, + identity, + }, + io, + )) } fn verify_file( - mut file: File, + file: File, digest: &str, expected_length: u64, diagnostic: &Path, ) -> Result<(), GfError> { + verify_file_counted(file, digest, expected_length, diagnostic).map(|_| ()) +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct ReadIoEvidence { + bytes: u64, + calls: u64, +} + +fn verify_file_counted( + mut file: File, + digest: &str, + expected_length: u64, + diagnostic: &Path, +) -> Result { let metadata = file .metadata() .map_err(|error| storage("inspect graph object handle", diagnostic, error))?; @@ -2753,6 +3010,7 @@ fn verify_file( )); } let mut hasher = Sha256::new(); + let mut io = ReadIoEvidence::default(); let mut buffer = vec![0_u8; BUFFER_BYTES]; loop { let read = file @@ -2761,12 +3019,14 @@ fn verify_file( if read == 0 { break; } + io.bytes = io.bytes.saturating_add(read as u64); + io.calls = io.calls.saturating_add(1); hasher.update(&buffer[..read]); } if hex_digest(hasher.finalize().into()) != digest { return Err(validation("graph object digest does not match its address")); } - Ok(()) + Ok(io) } fn verify_stream( @@ -2775,8 +3035,18 @@ fn verify_stream( expected_length: u64, diagnostic: &Path, ) -> Result<(), GfError> { + verify_stream_counted(file, digest, expected_length, diagnostic).map(|_| ()) +} + +fn verify_stream_counted( + file: &mut impl Read, + digest: &str, + expected_length: u64, + diagnostic: &Path, +) -> Result { let mut hasher = Sha256::new(); let mut total = 0_u64; + let mut calls = 0_u64; let mut buffer = vec![0_u8; BUFFER_BYTES]; loop { let read = file @@ -2786,12 +3056,16 @@ fn verify_stream( break; } total = total.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); + calls = calls.saturating_add(1); hasher.update(&buffer[..read]); } if total != expected_length || hex_digest(hasher.finalize().into()) != digest { return Err(validation("graph object digest does not match its address")); } - Ok(()) + Ok(ReadIoEvidence { + bytes: total, + calls, + }) } fn verify_and_seal_graph_object( @@ -2801,6 +3075,17 @@ fn verify_and_seal_graph_object( object_path: &Path, diagnostic: &Path, ) -> Result<(), GfError> { + verify_and_seal_graph_object_counted(file, digest, expected_length, object_path, diagnostic) + .map(|_| ()) +} + +fn verify_and_seal_graph_object_counted( + file: &File, + digest: &str, + expected_length: u64, + object_path: &Path, + diagnostic: &Path, +) -> Result { // Reuse is safe only after the exact opened inode is no longer writable. // Hashing first would leave a window in which the already-authenticated // bytes could be changed before the subsequent chmod. @@ -2818,7 +3103,7 @@ fn verify_and_seal_graph_object( return Err(validation("graph object is not canonically sealed")); } } - verify_file( + let io = verify_file_counted( file.try_clone() .map_err(|error| storage("clone graph object for authentication", diagnostic, error))?, digest, @@ -2835,7 +3120,7 @@ fn verify_and_seal_graph_object( "graph object became writable during authentication", )); } - Ok(()) + Ok(io) } #[cfg(unix)] @@ -3723,6 +4008,11 @@ mod tests { assert_eq!(evidence.files_reused, 1); assert_eq!(evidence.bytes_reused, payload.len() as u64); assert_eq!(evidence.files_copied, 0); + assert_eq!(evidence.application_read_bytes, payload.len() as u64); + assert_eq!(evidence.application_read_calls, 1); + assert_eq!(evidence.application_write_bytes, 0); + assert_eq!(evidence.application_write_calls, 0); + assert_eq!(evidence.fsync_calls, 0); } #[test] @@ -3762,6 +4052,17 @@ mod tests { assert_eq!(evidence.files_copied, inventory.file_count); assert_eq!(evidence.files_reused, 0); + let nonempty_bytes = inventory.total_byte_length; + let nonempty_files = inventory + .files + .iter() + .filter(|entry| entry.byte_length != 0) + .count() as u64; + assert_eq!(evidence.application_read_bytes, nonempty_bytes * 2); + assert_eq!(evidence.application_read_calls, nonempty_files * 2); + assert_eq!(evidence.application_write_bytes, nonempty_bytes); + assert_eq!(evidence.application_write_calls, nonempty_files); + assert_eq!(evidence.fsync_calls, inventory.file_count * 2); for entry in &inventory.files { let file = File::open(target.join(&entry.relative_path)).unwrap(); assert_eq!(graphforge_filesystem::file_link_count(&file).unwrap(), 1); @@ -3807,6 +4108,10 @@ mod tests { assert_eq!(first.bytes_hashed, 7); assert_eq!(first.bytes_installed, 7); assert!(!first.reused_existing); + assert_eq!(first.read_calls, 1); + assert_eq!(first.write_bytes, 7); + assert_eq!(first.write_calls, 1); + assert_eq!(first.fsync_calls, 3); assert!( root.path() .join(GRAPH_OBJECTS_DIR) @@ -3819,6 +4124,10 @@ mod tests { ); let (_, second) = install_graph_object_bytes(root.path(), b"payload").unwrap(); assert!(second.reused_existing); + assert_eq!(second.read_calls, 1); + assert_eq!(second.write_bytes, 0); + assert_eq!(second.write_calls, 0); + assert_eq!(second.fsync_calls, 0); assert_eq!( read_graph_object(root.path(), &digest, 7).unwrap(), b"payload" diff --git a/crates/graphforge-storage/src/lib.rs b/crates/graphforge-storage/src/lib.rs index 6c9c01979..79aafa2ef 100644 --- a/crates/graphforge-storage/src/lib.rs +++ b/crates/graphforge-storage/src/lib.rs @@ -19,6 +19,14 @@ pub mod filesystem_admission; pub mod adjacency; pub mod adjacency_delta; +pub mod storage_attribution; +pub use storage_attribution::{ + ArtifactCategory, ArtifactStorageTotals, ConstructionPhaseAttribution, PhaseIoTotals, + ProjectStorageIdentityUnion, StorageAllocationLifecycle, StorageAllocationTransition, + StorageAttributionSnapshot, StorageIoPhase, capture_project_storage_identity_union, + capture_storage_attribution, classify_graph_artifact, +}; + pub mod generation; pub use generation::{ commit_topology_aware, commit_topology_aware_with_auxiliary, read_search_generation, @@ -195,8 +203,8 @@ pub use project_portable_v2_export::{ plan_complete_portable_v2, plan_selected_portable_v2, repack_verified_expanded_portable_v2, }; pub use project_portable_v2_import::{ - PortableV2ImportPhase, PortableV2ImportProgress, PortableV2ImportReceipt, - PortableV2SelectiveCandidate, PortableV2StagedCompositionReceipt, + PortableV2ImportCleanupReceipt, PortableV2ImportPhase, PortableV2ImportProgress, + PortableV2ImportReceipt, PortableV2SelectiveCandidate, PortableV2StagedCompositionReceipt, consume_selective_portable_v2, import_complete_portable_v2, import_complete_portable_v2_with_progress, load_portable_ontology_staging, }; diff --git a/crates/graphforge-storage/src/project_portable_v2.rs b/crates/graphforge-storage/src/project_portable_v2.rs index 762ca3e4b..0f0ca2523 100644 --- a/crates/graphforge-storage/src/project_portable_v2.rs +++ b/crates/graphforge-storage/src/project_portable_v2.rs @@ -203,6 +203,16 @@ pub struct PortableV2Error { pub code: PortableV2ErrorCode, pub entry: Option, detail: &'static str, + /// Content-free native allocation evidence retained only for local + /// lifecycle qualification of an interrupted operation. + #[doc(hidden)] + pub allocation_identity_allocated_bytes: std::collections::BTreeMap, + /// Bytes actually read while reauthenticating an interrupted import. + #[doc(hidden)] + pub recovery_reauthentication_read_bytes: u64, + /// Calls actually completed while reauthenticating an interrupted import. + #[doc(hidden)] + pub recovery_reauthentication_read_calls: u64, } impl PortableV2Error { @@ -213,6 +223,9 @@ impl PortableV2Error { code, entry: None, detail, + allocation_identity_allocated_bytes: std::collections::BTreeMap::new(), + recovery_reauthentication_read_bytes: 0, + recovery_reauthentication_read_calls: 0, } } pub(crate) fn at(code: PortableV2ErrorCode, entry: &str, detail: &'static str) -> Self { @@ -220,8 +233,29 @@ impl PortableV2Error { code, entry: Some(entry.chars().take(4096).collect()), detail, + allocation_identity_allocated_bytes: std::collections::BTreeMap::new(), + recovery_reauthentication_read_bytes: 0, + recovery_reauthentication_read_calls: 0, } } + + pub(crate) fn with_allocation_identities( + mut self, + identities: std::collections::BTreeMap, + ) -> Self { + self.allocation_identity_allocated_bytes = identities; + self + } + + pub(crate) fn with_recovery_reauthentication( + mut self, + read_bytes: u64, + read_calls: u64, + ) -> Self { + self.recovery_reauthentication_read_bytes = read_bytes; + self.recovery_reauthentication_read_calls = read_calls; + self + } } impl fmt::Display for PortableV2Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -385,9 +419,9 @@ pub fn verify_portable_v2( ) })?; if metadata.is_dir() { - materialize_expanded(source, staging.path(), limits, cancelled)?; + materialize_expanded(source, staging.path(), limits, cancelled, &mut |_| Ok(()))?; } else { - materialize_bundle(source, staging.path(), limits, cancelled)?; + materialize_bundle(source, staging.path(), limits, cancelled, &mut |_| Ok(()))?; } validate_materialized_ontology_composition(staging.path(), &report, limits, cancelled)?; } @@ -839,6 +873,23 @@ pub fn materialize_verified_portable_v2( limits: PortableV2Limits, cancelled: Option<&AtomicBool>, ) -> Result { + materialize_verified_portable_v2_observed(source, destination, limits, cancelled, |_| Ok(())) + .map(|materialized| materialized.report) +} + +pub(crate) struct VerifiedMaterialization { + pub(crate) report: PortableV2Report, + pub(crate) application_read_bytes: u64, + pub(crate) application_read_operations: u64, +} + +pub(crate) fn materialize_verified_portable_v2_observed( + source: impl AsRef, + destination: impl AsRef, + limits: PortableV2Limits, + cancelled: Option<&AtomicBool>, + mut observed: impl FnMut(&File) -> Result<(), PortableV2Error>, +) -> Result { let source = source.as_ref(); let destination = destination.as_ref(); if destination.exists() { @@ -854,14 +905,17 @@ pub fn materialize_verified_portable_v2( PortableV2Error::new(PortableV2ErrorCode::Io, "cannot create materialization") })?; let result = if before.is_dir() { - materialize_expanded(source, destination, limits, cancelled) + materialize_expanded(source, destination, limits, cancelled, &mut observed) } else { - materialize_bundle(source, destination, limits, cancelled) + materialize_bundle(source, destination, limits, cancelled, &mut observed) + }; + let (application_read_bytes, application_read_operations) = match result { + Ok(stats) => stats, + Err(error) => { + let _ = fs::remove_dir_all(destination); + return Err(error); + } }; - if let Err(error) = result { - let _ = fs::remove_dir_all(destination); - return Err(error); - } let after = fs::metadata(source).map_err(|_| { PortableV2Error::new( PortableV2ErrorCode::ConcurrentMutation, @@ -898,7 +952,11 @@ pub fn materialize_verified_portable_v2( "source changed during materialization", )); } - Ok(report) + Ok(VerifiedMaterialization { + report, + application_read_bytes, + application_read_operations, + }) } fn materialize_expanded( @@ -906,7 +964,10 @@ fn materialize_expanded( destination: &Path, limits: PortableV2Limits, cancelled: Option<&AtomicBool>, -) -> Result<(), PortableV2Error> { + observed: &mut impl FnMut(&File) -> Result<(), PortableV2Error>, +) -> Result<(u64, u64), PortableV2Error> { + let mut read_bytes = 0_u64; + let mut read_operations = 0_u64; let mut paths = Vec::new(); walk(source, source, &mut paths, limits, cancelled)?; for relative in paths @@ -930,10 +991,14 @@ fn materialize_expanded( .map_err(|_| { PortableV2Error::at(PortableV2ErrorCode::Io, &relative, "cannot stage entry") })?; - copy_materialized(&mut input, &mut output, limits.copy_buffer_bytes, cancelled)?; + let (bytes, operations) = + copy_materialized(&mut input, &mut output, limits.copy_buffer_bytes, cancelled)?; + read_bytes = read_bytes.saturating_add(bytes); + read_operations = read_operations.saturating_add(operations); output.sync_all().map_err(|_| { PortableV2Error::at(PortableV2ErrorCode::Io, &relative, "cannot sync entry") })?; + observed(&output)?; let after = fs::metadata(&input_path).map_err(|_| { PortableV2Error::at( PortableV2ErrorCode::ConcurrentMutation, @@ -949,7 +1014,8 @@ fn materialize_expanded( )); } } - sync_materialized_tree(destination) + sync_materialized_tree(destination)?; + Ok((read_bytes, read_operations)) } fn materialize_bundle( @@ -957,7 +1023,10 @@ fn materialize_bundle( destination: &Path, limits: PortableV2Limits, cancelled: Option<&AtomicBool>, -) -> Result<(), PortableV2Error> { + observed: &mut impl FnMut(&File) -> Result<(), PortableV2Error>, +) -> Result<(u64, u64), PortableV2Error> { + let mut read_bytes = 0_u64; + let mut read_operations = 0_u64; let mut input = File::open(source) .map_err(|_| PortableV2Error::new(PortableV2ErrorCode::Io, "cannot reopen bundle"))?; let mut pending_pax = None; @@ -998,22 +1067,26 @@ fn materialize_bundle( .map_err(|_| { PortableV2Error::at(PortableV2ErrorCode::Io, &path, "cannot stage entry") })?; - copy_exact_materialized( + let (bytes, operations) = copy_exact_materialized( &mut input, &mut output, size, limits.copy_buffer_bytes, cancelled, )?; + read_bytes = read_bytes.saturating_add(bytes); + read_operations = read_operations.saturating_add(operations); output.sync_all().map_err(|_| { PortableV2Error::at(PortableV2ErrorCode::Io, &path, "cannot sync entry") })?; + observed(&output)?; } else { skip_exact(&mut input, size, limits.copy_buffer_bytes, cancelled)?; } skip_padding(&mut input, size)?; } - sync_materialized_tree(destination) + sync_materialized_tree(destination)?; + Ok((read_bytes, read_operations)) } fn create_materialized_parent(path: &Path, entry: &str) -> Result<(), PortableV2Error> { @@ -1030,16 +1103,20 @@ fn copy_materialized( output: &mut impl Write, buffer_size: usize, cancelled: Option<&AtomicBool>, -) -> Result<(), PortableV2Error> { +) -> Result<(u64, u64), PortableV2Error> { let mut buffer = vec![0; buffer_size]; + let mut bytes = 0_u64; + let mut operations = 0_u64; loop { check_cancel(cancelled)?; let count = input .read(&mut buffer) .map_err(|_| PortableV2Error::new(PortableV2ErrorCode::Io, "cannot read entry"))?; if count == 0 { - return Ok(()); + return Ok((bytes, operations)); } + bytes = bytes.saturating_add(count as u64); + operations = operations.saturating_add(1); output .write_all(&buffer[..count]) .map_err(|_| PortableV2Error::new(PortableV2ErrorCode::Io, "cannot stage entry"))?; @@ -1051,7 +1128,7 @@ fn copy_exact_materialized( length: u64, buffer_size: usize, cancelled: Option<&AtomicBool>, -) -> Result<(), PortableV2Error> { +) -> Result<(u64, u64), PortableV2Error> { let mut remaining = length; let mut buffer = vec![0; buffer_size]; while remaining > 0 { @@ -1065,7 +1142,8 @@ fn copy_exact_materialized( .map_err(|_| PortableV2Error::new(PortableV2ErrorCode::Io, "cannot stage entry"))?; remaining -= count as u64; } - Ok(()) + let operations = length.div_ceil(buffer_size as u64); + Ok((length, operations)) } fn skip_exact( input: &mut File, @@ -1074,7 +1152,7 @@ fn skip_exact( cancelled: Option<&AtomicBool>, ) -> Result<(), PortableV2Error> { let mut sink = std::io::sink(); - copy_exact_materialized(input, &mut sink, length, buffer_size, cancelled) + copy_exact_materialized(input, &mut sink, length, buffer_size, cancelled).map(|_| ()) } fn skip_padding(input: &mut File, length: u64) -> Result<(), PortableV2Error> { let padding = (512 - length % 512) % 512; @@ -3510,4 +3588,17 @@ mod tests { assert_eq!(bundled.representation, PortableV2Representation::Bundle); assert_ne!(expanded.transport_digest, bundled.transport_digest); } + + #[test] + fn materialization_reports_actual_bounded_payload_reads() { + let parent = tempfile::tempdir().unwrap(); + let input_path = parent.path().join("input"); + fs::write(&input_path, vec![7_u8; 10]).unwrap(); + let mut input = File::open(input_path).unwrap(); + let mut output = Vec::new(); + let (bytes, operations) = + copy_exact_materialized(&mut input, &mut output, 10, 4, None).unwrap(); + assert_eq!((bytes, operations), (10, 3)); + assert_eq!(output, vec![7_u8; 10]); + } } diff --git a/crates/graphforge-storage/src/project_portable_v2_export.rs b/crates/graphforge-storage/src/project_portable_v2_export.rs index d67cd9ca8..bb5629bcb 100644 --- a/crates/graphforge-storage/src/project_portable_v2_export.rs +++ b/crates/graphforge-storage/src/project_portable_v2_export.rs @@ -1,6 +1,6 @@ //! Bounded deterministic portable-project v2 complete-package export. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::{self, File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; @@ -355,7 +355,7 @@ impl PortableV2ExportPlan { Ok(()) } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] /// Durable publication receipt with separate semantic and transport identities. pub struct PortableV2ExportReceipt { /// Pinned source generation. @@ -372,6 +372,53 @@ pub struct PortableV2ExportReceipt { pub output: PortableV2Output, /// Fingerprint of the immutable content-free selection preview used by the writer. pub selection_fingerprint: String, + /// Exact native allocation of the published package for lifecycle evidence. + #[doc(hidden)] + pub allocation_identity_allocated_bytes: BTreeMap, + /// Logical EOF bytes of the exact published identity union. + #[doc(hidden)] + pub allocation_logical_bytes: u64, + /// Distinct physical files in the exact published identity union. + #[doc(hidden)] + pub allocation_physical_objects: u64, +} + +impl PartialEq for PortableV2ExportReceipt { + fn eq(&self, other: &Self) -> bool { + self.generation_uuid == other.generation_uuid + && self.package_digest == other.package_digest + && self.transport_digest == other.transport_digest + && self.entry_count == other.entry_count + && self.payload_bytes == other.payload_bytes + && self.output == other.output + && self.selection_fingerprint == other.selection_fingerprint + && self.allocation_logical_bytes == other.allocation_logical_bytes + && self.allocation_physical_objects == other.allocation_physical_objects + } +} + +impl Eq for PortableV2ExportReceipt {} + +#[derive(Default)] +struct ExportAllocationObserver { + allocated: BTreeMap, + logical: BTreeMap, +} + +impl ExportAllocationObserver { + fn observe(&mut self, file: &File) -> Result<(), ExportError> { + let identity = graphforge_filesystem::file_identity(file).map_err(storage)?; + let usage = graphforge_filesystem::file_space_usage(file).map_err(storage)?; + let mut file_id = String::with_capacity(32); + for byte in identity.file_id { + use std::fmt::Write as _; + write!(&mut file_id, "{byte:02x}").expect("writing to String cannot fail"); + } + let key = format!("{:016x}:{file_id}", identity.volume_serial); + self.allocated.insert(key.clone(), usage.allocated_bytes); + self.logical.insert(key, usage.logical_bytes); + Ok(()) + } } #[derive(Serialize)] @@ -1104,23 +1151,45 @@ pub fn export_complete_portable_v2( .ok_or_else(|| err("GF_INVALID_DESTINATION", "invalid destination name"))?; let stage = parent.join(format!(".{name}.{}.partial", Uuid::new_v4())); let is_cancelled = || cancelled.load(Ordering::Relaxed); + let mut allocation = ExportAllocationObserver::default(); let result = match output { - PortableV2Output::Expanded => expanded(plan, &stage, limits, &is_cancelled, &mut progress), - PortableV2Output::Bundle => bundle(plan, &stage, limits, &is_cancelled, &mut progress), + PortableV2Output::Expanded => expanded( + plan, + &stage, + limits, + &is_cancelled, + &mut progress, + &mut allocation, + ), + PortableV2Output::Bundle => bundle( + plan, + &stage, + limits, + &is_cancelled, + &mut progress, + &mut allocation, + ), }; let digest = match result { Ok(d) => d, Err(e) => { remove(&stage); - return Err(e); + return Err(e.with_allocation_identities(allocation.allocated)); } }; + let allocation_logical_bytes = allocation.logical.values().copied().sum(); + let allocation_physical_objects = allocation.logical.len() as u64; + let staged_allocation = allocation.allocated; if is_cancelled() { remove(&stage); - return Err(err("GF_CANCELLED", "portable export cancelled")); + return Err(err("GF_CANCELLED", "portable export cancelled") + .with_allocation_identities(staged_allocation)); } let verified = verify_portable_v2(&stage, PortableV2Mode::Full, limits, Some(cancelled)) - .inspect_err(|_| remove(&stage))?; + .map_err(|error| { + remove(&stage); + error.with_allocation_identities(staged_allocation.clone()) + })?; let expected_transport = format!("sha256:{}", hex(digest)); if verified.package_class != plan.package_class || verified.package_digest != format!("sha256:{}", hex(plan.package_digest)) @@ -1129,22 +1198,24 @@ pub fn export_complete_portable_v2( return Err(PortableV2Error::new( PortableV2ErrorCode::DigestMismatch, "writer and verifier semantic receipts disagree", - )); + ) + .with_allocation_identities(staged_allocation.clone())); } if verified.transport_digest.as_deref() != Some(expected_transport.as_str()) { remove(&stage); return Err(PortableV2Error::new( PortableV2ErrorCode::DigestMismatch, "writer and verifier transport receipts disagree", - )); + ) + .with_allocation_identities(staged_allocation.clone())); } publish_no_replace(&stage, dst).map_err(|error| { remove(&stage); - storage(error) + storage(error).with_allocation_identities(staged_allocation.clone()) })?; if let Err(error) = sync_dir(parent) { remove(dst); - return Err(error); + return Err(error.with_allocation_identities(staged_allocation)); } Ok(PortableV2ExportReceipt { generation_uuid: plan.generation_uuid, @@ -1155,6 +1226,9 @@ pub fn export_complete_portable_v2( payload_bytes: plan.payload_bytes, output, selection_fingerprint: plan.selection_fingerprint.clone(), + allocation_identity_allocated_bytes: staged_allocation, + allocation_logical_bytes, + allocation_physical_objects, }) } @@ -1299,9 +1373,15 @@ fn expanded( l: PortableV2ExportLimits, cancelled: &impl Fn() -> bool, progress: &mut impl FnMut(PortableV2ExportProgress), + allocation: &mut ExportAllocationObserver, ) -> Result<[u8; 32], ExportError> { fs::create_dir(stage).map_err(storage)?; - write_bytes(stage, "data/graphforge-project.json", &plan.manifest)?; + write_bytes( + stage, + "data/graphforge-project.json", + &plan.manifest, + allocation, + )?; let mut payload = vec![( "data/graphforge-project.json".into(), plan.manifest.len() as u64, @@ -1311,15 +1391,22 @@ fn expanded( for (i, f) in plan.files.iter().enumerate() { let target = stage.join(&f.path); parent(&target)?; - copy(f, &target, l.copy_buffer_bytes, cancelled, |n| { - done += n; - progress(PortableV2ExportProgress { - entries_completed: i + 1, - bytes_completed: done, - entries_total: plan.files.len() + 5, - bytes_total: plan.payload_bytes, - }); - })?; + copy( + f, + &target, + l.copy_buffer_bytes, + cancelled, + allocation, + |n| { + done += n; + progress(PortableV2ExportProgress { + entries_completed: i + 1, + bytes_completed: done, + entries_total: plan.files.len() + 5, + bytes_total: plan.payload_bytes, + }); + }, + )?; progress(PortableV2ExportProgress { entries_completed: i + 2, bytes_completed: done, @@ -1330,9 +1417,9 @@ fn expanded( } payload.sort_by(|a, b| a.0.cmp(&b.0)); let inv = inventory(&payload, l.max_tag_manifest_bytes)?; - write_bytes(stage, "manifest-sha256.txt", &inv)?; - write_bytes(stage, "bagit.txt", BAGIT)?; - write_bytes(stage, "bag-info.txt", BAG_INFO)?; + write_bytes(stage, "manifest-sha256.txt", &inv, allocation)?; + write_bytes(stage, "bagit.txt", BAGIT, allocation)?; + write_bytes(stage, "bag-info.txt", BAG_INFO, allocation)?; let tags = [ ("bag-info.txt", BAG_INFO), ("bagit.txt", BAGIT), @@ -1343,7 +1430,7 @@ fn expanded( .map(|(p, b)| (p.to_string(), b.len() as u64, Sha256::digest(b).into())) .collect::>(); let tag = inventory(&tag_rows, l.max_tag_manifest_bytes)?; - write_bytes(stage, "tagmanifest-sha256.txt", &tag)?; + write_bytes(stage, "tagmanifest-sha256.txt", &tag, allocation)?; progress(PortableV2ExportProgress { entries_completed: plan.files.len() + 5, bytes_completed: done, @@ -1425,6 +1512,7 @@ fn bundle( l: PortableV2ExportLimits, cancelled: &impl Fn() -> bool, progress: &mut impl FnMut(PortableV2ExportProgress), + allocation: &mut ExportAllocationObserver, ) -> Result<[u8; 32], ExportError> { let mut items = entries(plan, l.max_tag_manifest_bytes)?; items.sort_by(|a, b| a.0.cmp(&b.0)); @@ -1440,19 +1528,32 @@ fn bundle( return Err(err("GF_CANCELLED", "portable export cancelled")); } header(&mut out, &mut h, path, src.len())?; + allocation.observe(&out)?; match src { - Src::Bytes(b) => emit(&mut out, &mut h, b)?, - Src::File(f) => stream(&mut out, &mut h, f, l.copy_buffer_bytes, cancelled, |n| { - done += n; - progress(PortableV2ExportProgress { - entries_completed: i, - bytes_completed: done, - entries_total: items.len(), - bytes_total: plan.payload_bytes, - }); - })?, + Src::Bytes(b) => { + emit(&mut out, &mut h, b)?; + allocation.observe(&out)?; + } + Src::File(f) => stream( + &mut out, + &mut h, + f, + l.copy_buffer_bytes, + cancelled, + allocation, + |n| { + done += n; + progress(PortableV2ExportProgress { + entries_completed: i, + bytes_completed: done, + entries_total: items.len(), + bytes_total: plan.payload_bytes, + }); + }, + )?, } pad(&mut out, &mut h, src.len())?; + allocation.observe(&out)?; progress(PortableV2ExportProgress { entries_completed: i + 1, bytes_completed: done, @@ -1462,8 +1563,10 @@ fn bundle( } let end = [0u8; 1024]; out.write_all(&end).map_err(storage)?; + allocation.observe(&out)?; h.update(end); out.sync_all().map_err(storage)?; + allocation.observe(&out)?; Ok(h.finalize().into()) } @@ -1570,6 +1673,7 @@ fn copy( target: &Path, size: usize, cancelled: &impl Fn() -> bool, + allocation: &mut ExportAllocationObserver, mut tick: impl FnMut(u64), ) -> Result<(), ExportError> { let mut output = OpenOptions::new() @@ -1582,7 +1686,9 @@ fn copy( return Err(err("GF_CANCELLED", "portable export cancelled")); } output.write_all(bytes).map_err(storage)?; + allocation.observe(&output)?; output.sync_all().map_err(storage)?; + allocation.observe(&output)?; tick(bytes.len() as u64); return Ok(()); } @@ -1604,6 +1710,7 @@ fn copy( tick(count as u64); } output.sync_all().map_err(storage)?; + allocation.observe(&output)?; if bytes_read != planned.length || <[u8; 32]>::from(digest.finalize()) != planned.digest { return Err(err("GF_SOURCE_CHANGED", "source changed during export")); } @@ -1620,6 +1727,7 @@ fn stream( planned: &PlannedFile, size: usize, cancelled: &impl Fn() -> bool, + allocation: &mut ExportAllocationObserver, mut tick: impl FnMut(u64), ) -> Result<(), ExportError> { if let PlannedSource::Control(bytes) = &planned.source { @@ -1627,6 +1735,7 @@ fn stream( return Err(err("GF_CANCELLED", "portable export cancelled")); } out.write_all(bytes).map_err(storage)?; + allocation.observe(out)?; transport.update(bytes); tick(bytes.len() as u64); return Ok(()); @@ -2077,7 +2186,12 @@ fn publish_no_replace(_: &Path, _: &Path) -> std::io::Result<()> { fn parent(p: &Path) -> Result<(), ExportError> { fs::create_dir_all(p.parent().unwrap()).map_err(storage) } -fn write_bytes(root: &Path, p: &str, b: &[u8]) -> Result<(), ExportError> { +fn write_bytes( + root: &Path, + p: &str, + b: &[u8], + allocation: &mut ExportAllocationObserver, +) -> Result<(), ExportError> { let p = root.join(p); parent(&p)?; let mut f = OpenOptions::new() @@ -2086,7 +2200,9 @@ fn write_bytes(root: &Path, p: &str, b: &[u8]) -> Result<(), ExportError> { .open(p) .map_err(storage)?; f.write_all(b).map_err(storage)?; - f.sync_all().map_err(storage) + allocation.observe(&f)?; + f.sync_all().map_err(storage)?; + allocation.observe(&f) } fn sync_tree(root: &Path) -> Result<(), ExportError> { let mut dirs = vec![root.into()]; @@ -2595,8 +2711,25 @@ mod tests { let expanded_path = root.join("hostile.gfproject"); let bundle_path = root.join("hostile.gfpb"); let limits = PortableV2ExportLimits::default(); - expanded(plan, &expanded_path, limits, &|| false, &mut |_| {}).unwrap(); - bundle(plan, &bundle_path, limits, &|| false, &mut |_| {}).unwrap(); + let mut allocation = ExportAllocationObserver::default(); + expanded( + plan, + &expanded_path, + limits, + &|| false, + &mut |_| {}, + &mut allocation, + ) + .unwrap(); + bundle( + plan, + &bundle_path, + limits, + &|| false, + &mut |_| {}, + &mut allocation, + ) + .unwrap(); (expanded_path, bundle_path) } @@ -2631,6 +2764,15 @@ mod tests { expanded_receipt.package_digest, bundle_receipt.package_digest ); + assert!( + expanded_receipt.allocation_identity_allocated_bytes.len() > 1, + "expanded writer must report each exact published identity" + ); + assert_eq!( + bundle_receipt.allocation_identity_allocated_bytes.len(), + 1, + "bundle writer must report its one exact published identity" + ); let expanded_report = verify_portable_v2(&expanded, PortableV2Mode::Full, limits, Some(&cancelled)).unwrap(); let bundle_report = @@ -3926,6 +4068,10 @@ mod tests { ) .unwrap_err(); assert_eq!(error.code, PortableV2ErrorCode::ConcurrentMutation); + assert!( + !error.allocation_identity_allocated_bytes.is_empty(), + "partial bundle allocation must survive typed failure" + ); assert!(!mutated.exists()); } @@ -3998,11 +4144,13 @@ mod tests { assert_eq!(total, 32 * 1024 * 1024); let destination = root.path().join("dense.parquet"); let mut observed = 0; + let mut allocation = ExportAllocationObserver::default(); copy( &planned, &destination, limits.copy_buffer_bytes, &|| false, + &mut allocation, |bytes| { observed += bytes; }, diff --git a/crates/graphforge-storage/src/project_portable_v2_import.rs b/crates/graphforge-storage/src/project_portable_v2_import.rs index b0704d880..945daf701 100644 --- a/crates/graphforge-storage/src/project_portable_v2_import.rs +++ b/crates/graphforge-storage/src/project_portable_v2_import.rs @@ -36,6 +36,27 @@ pub struct PortableV2ImportReceipt { pub publication: ProjectPublicationReceipt, /// Durable non-authoritative composition candidate, when imported. pub staged_composition: Option, + /// Exact native identities simultaneously retained by private materialization. + pub materialized_identity_allocated_bytes: std::collections::BTreeMap, + /// Exact authenticated identity union of the published project container, + /// including controls and every retained generation. + pub published_identity_allocated_bytes: std::collections::BTreeMap, + /// Identity-safe, durably synchronized removal of private import materialization. + pub materialized_cleanup: PortableV2ImportCleanupReceipt, +} + +/// Exact cleanup receipt for private portable-import materialization. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PortableV2ImportCleanupReceipt { + /// Native identities confirmed removed from the private staging owner. + pub removed_identity_allocated_bytes: std::collections::BTreeMap, + /// The containing namespace was synchronized after removal. + pub parent_sync_confirmed: bool, +} + +#[cfg(test)] +thread_local! { + static INJECT_IMPORT_CLEANUP_FAILURE: std::cell::Cell = const { std::cell::Cell::new(false) }; } #[derive(Debug, Clone, PartialEq, Eq)] @@ -214,34 +235,31 @@ pub fn import_complete_portable_v2_with_progress( bytes: 0, package_digest: None, }); - let target_name = target - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| { - PortableV2Error::new(PortableV2ErrorCode::InvalidPath, "invalid import target") - })?; - let stage = target - .parent() - .unwrap_or_else(|| Path::new(".")) - .join(format!( - ".{target_name}.portable-v2-{}", - transaction_uuid.hyphenated() - )); - let (owner, owned_retry) = claim_stage(&stage, target_name, transaction_uuid, generation_uuid)?; - let report = match materialize_verified_portable_v2(source, &stage, limits, cancelled) { - Ok(report) => report, - Err(error) => { - let _ = fs::remove_file(&owner); - let _ = sync_parent(&owner); - return Err(error); - } - }; + let OwnedMaterialization { + stage, + owner, + owned_retry, + identities: materialized_identity_allocated_bytes, + report, + materialization_read_bytes, + materialization_read_operations, + stage_identity: materialized_stage_identity, + entry_count, + } = materialize_owned_import( + source, + target, + transaction_uuid, + generation_uuid, + limits, + cancelled, + )?; progress(PortableV2ImportProgress { phase: PortableV2ImportPhase::Materialized, entries: report.entry_count, bytes: report.payload_bytes, package_digest: Some(report.package_digest.clone()), }); + let allocation_on_error = materialized_identity_allocated_bytes.clone(); let result = import_materialized( &stage, target, @@ -252,10 +270,53 @@ pub fn import_complete_portable_v2_with_progress( cancelled, &report, owned_retry, - ); - let _ = fs::remove_dir_all(&stage); - let _ = fs::remove_file(&owner); - let _ = sync_parent(&owner); + ) + .map(|mut receipt| { + receipt.materialized_identity_allocated_bytes = materialized_identity_allocated_bytes; + receipt + }) + .map_err(|error| { + let mut owned_identities = allocation_on_error; + if let Err(cleanup_error) = cleanup_failed_import_finalization( + &stage, + &owner, + materialized_stage_identity, + &mut owned_identities, + entry_count, + ) { + return cleanup_error.with_allocation_identities(owned_identities); + } + error + .with_allocation_identities(owned_identities) + // Preserve the actual bounded payload-copy reads completed before + // finalization failed instead of approximating them from entries. + .with_recovery_reauthentication( + materialization_read_bytes, + materialization_read_operations, + ) + }); + let result = result.and_then(|mut receipt| { + // Finalization can create additional authenticated composition files in + // staging. Add their identities to the operation-wide owned union + // immediately before cleanup; identities of atomically replaced files + // remain attributable even though they are no longer live. + capture_finalized_import_identities( + &stage, + materialized_stage_identity, + &mut receipt.materialized_identity_allocated_bytes, + entry_count, + )?; + receipt.materialized_cleanup = cleanup_import_materialization( + &stage, + &owner, + materialized_stage_identity, + &receipt.materialized_identity_allocated_bytes, + ) + .map_err(|error| { + error.with_allocation_identities(receipt.materialized_identity_allocated_bytes.clone()) + })?; + Ok(receipt) + }); if result.is_ok() { progress(PortableV2ImportProgress { phase: PortableV2ImportPhase::Published, @@ -267,6 +328,299 @@ pub fn import_complete_portable_v2_with_progress( result } +struct OwnedMaterialization { + stage: PathBuf, + owner: PathBuf, + owned_retry: bool, + identities: std::collections::BTreeMap, + report: PortableV2Report, + materialization_read_bytes: u64, + materialization_read_operations: u64, + stage_identity: graphforge_filesystem::FileIdentity, + entry_count: usize, +} + +fn materialize_owned_import( + source: &Path, + target: &Path, + transaction_uuid: Uuid, + generation_uuid: Uuid, + limits: PortableV2Limits, + cancelled: Option<&AtomicBool>, +) -> Result { + let target_name = target + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + PortableV2Error::new(PortableV2ErrorCode::InvalidPath, "invalid import target") + })?; + let stage = target + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(format!( + ".{target_name}.portable-v2-{}", + transaction_uuid.hyphenated() + )); + let (owner, owned_retry) = claim_stage(&stage, target_name, transaction_uuid, generation_uuid)?; + let mut identities = std::collections::BTreeMap::new(); + let owner_file = fs::File::open(&owner).map_err(|_| { + PortableV2Error::new(PortableV2ErrorCode::Io, "cannot open import ownership") + })?; + record_import_file_identity(&owner_file, &mut identities)?; + let materialized = match crate::project_portable_v2::materialize_verified_portable_v2_observed( + source, + &stage, + limits, + cancelled, + |file| record_import_file_identity(file, &mut identities), + ) { + Ok(materialized) => materialized, + Err(error) => { + let _ = fs::remove_file(&owner); + let _ = sync_parent(&owner); + return Err(error.with_allocation_identities(identities)); + } + }; + let report = materialized.report; + // Atomic replacement can change identities after the write observer. The + // completed boundary is the cleanup authority; the later finalization + // capture extends this into the operation-wide identity union. + identities.clear(); + record_import_file_identity(&owner_file, &mut identities)?; + let stage_directory = graphforge_filesystem::StableDirectory::open(&stage).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot authenticate completed import staging", + ) + })?; + let entry_count = usize::try_from(report.entry_count).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::LimitExceeded, + "import entry count exceeds platform capacity", + ) + })?; + let mut capture_budget = entry_count.saturating_mul(2).saturating_add(1024); + capture_import_tree(&stage_directory, &mut identities, &mut capture_budget)?; + Ok(OwnedMaterialization { + stage, + owner, + owned_retry, + identities, + report, + materialization_read_bytes: materialized.application_read_bytes, + materialization_read_operations: materialized.application_read_operations, + stage_identity: stage_directory.identity(), + entry_count, + }) +} + +fn capture_finalized_import_identities( + stage: &Path, + expected_stage_identity: graphforge_filesystem::FileIdentity, + identities: &mut std::collections::BTreeMap, + entry_count: usize, +) -> Result<(), PortableV2Error> { + let directory = graphforge_filesystem::StableDirectory::open(stage).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot authenticate finalized import staging", + ) + })?; + if directory.identity() != expected_stage_identity { + return Err(PortableV2Error::new( + PortableV2ErrorCode::Io, + "import staging identity changed during finalization", + )); + } + let mut capture_budget = entry_count.saturating_mul(2).saturating_add(1024); + capture_import_tree(&directory, identities, &mut capture_budget) +} + +fn cleanup_failed_import_finalization( + stage: &Path, + owner: &Path, + expected_stage_identity: graphforge_filesystem::FileIdentity, + identities: &mut std::collections::BTreeMap, + entry_count: usize, +) -> Result<(), PortableV2Error> { + capture_finalized_import_identities(stage, expected_stage_identity, identities, entry_count)?; + cleanup_import_materialization(stage, owner, expected_stage_identity, identities).map(|_| ()) +} + +fn cleanup_import_materialization( + stage: &Path, + owner: &Path, + expected_stage_identity: graphforge_filesystem::FileIdentity, + identities: &std::collections::BTreeMap, +) -> Result { + #[cfg(test)] + if INJECT_IMPORT_CLEANUP_FAILURE.with(std::cell::Cell::get) { + return Err(PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot durably clean import staging", + )); + } + let mut removed_identities = std::collections::BTreeMap::new(); + if stage.exists() { + let parent = graphforge_filesystem::StableDirectory::open( + stage.parent().unwrap_or_else(|| Path::new(".")), + ) + .map_err(|_| { + PortableV2Error::new(PortableV2ErrorCode::Io, "cannot open import staging parent") + })?; + let name = stage.file_name().ok_or_else(|| { + PortableV2Error::new( + PortableV2ErrorCode::InvalidPath, + "invalid import staging path", + ) + })?; + let directory = parent.open_child_directory(name).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot authenticate import staging", + ) + })?; + if directory.identity() != expected_stage_identity { + return Err(PortableV2Error::new( + PortableV2ErrorCode::Io, + "import staging identity changed before cleanup", + )); + } + let mut cleanup_budget = identities.len().saturating_mul(2).saturating_add(1024); + remove_stable_tree( + &directory, + identities, + &mut removed_identities, + &mut cleanup_budget, + )?; + parent + .remove_child_directory_if_identity(name, directory.identity()) + .map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot remove authenticated import staging", + ) + })?; + parent.sync().map_err(|_| { + PortableV2Error::new(PortableV2ErrorCode::Io, "cannot sync import staging parent") + })?; + } + if owner.exists() { + let parent = graphforge_filesystem::StableDirectory::open( + owner.parent().unwrap_or_else(|| Path::new(".")), + ) + .map_err(|_| { + PortableV2Error::new(PortableV2ErrorCode::Io, "cannot open import owner parent") + })?; + let name = owner.file_name().ok_or_else(|| { + PortableV2Error::new( + PortableV2ErrorCode::InvalidPath, + "invalid import owner path", + ) + })?; + let file = parent.open_child_file(name).map_err(|_| { + PortableV2Error::new(PortableV2ErrorCode::Io, "cannot authenticate import owner") + })?; + let identity = graphforge_filesystem::file_identity(&file).map_err(|_| { + PortableV2Error::new(PortableV2ErrorCode::Io, "cannot identify import owner") + })?; + let mut observed = std::collections::BTreeMap::new(); + record_import_file_identity(&file, &mut observed)?; + if observed + .keys() + .any(|identity| !identities.contains_key(identity)) + { + return Err(PortableV2Error::new( + PortableV2ErrorCode::Io, + "import owner identity changed before cleanup", + )); + } + removed_identities.extend(observed); + parent + .unlink_child_if_identity(name, identity) + .map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot remove authenticated import owner", + ) + })?; + parent.sync().map_err(|_| { + PortableV2Error::new(PortableV2ErrorCode::Io, "cannot sync import owner parent") + })?; + } + Ok(PortableV2ImportCleanupReceipt { + removed_identity_allocated_bytes: removed_identities, + parent_sync_confirmed: true, + }) +} + +fn remove_stable_tree( + directory: &graphforge_filesystem::StableDirectory, + identities: &std::collections::BTreeMap, + removed_identities: &mut std::collections::BTreeMap, + remaining: &mut usize, +) -> Result<(), PortableV2Error> { + let names = directory.child_names_bounded(*remaining).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "import staging cleanup exceeds bound", + ) + })?; + *remaining = (*remaining).saturating_sub(names.len()); + for name in names { + if let Ok(child) = directory.open_child_directory(&name) { + remove_stable_tree(&child, identities, removed_identities, remaining)?; + directory + .remove_child_directory_if_identity(&name, child.identity()) + .map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot remove authenticated import directory", + ) + })?; + } else { + let file = directory.open_child_file(&name).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot authenticate import cleanup entry", + ) + })?; + let identity = graphforge_filesystem::file_identity(&file).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot identify import cleanup entry", + ) + })?; + let mut observed = std::collections::BTreeMap::new(); + record_import_file_identity(&file, &mut observed)?; + if observed + .keys() + .any(|identity| !identities.contains_key(identity)) + { + return Err(PortableV2Error::new( + PortableV2ErrorCode::Io, + "import cleanup entry is not owned materialization", + )); + } + removed_identities.extend(observed); + directory + .unlink_child_if_identity(&name, identity) + .map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot remove authenticated import entry", + ) + })?; + } + } + directory.sync().map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot sync authenticated import staging", + ) + }) +} + fn claim_stage( stage: &Path, target_name: &str, @@ -586,9 +940,73 @@ fn import_materialized( transport_digest: report.transport_digest.clone(), publication, staged_composition, + materialized_identity_allocated_bytes: std::collections::BTreeMap::new(), + published_identity_allocated_bytes: crate::capture_project_storage_identity_union( + &reopened, + ) + .map_err(storage)? + .physical_identity_allocated_bytes, + materialized_cleanup: PortableV2ImportCleanupReceipt::default(), }) } +fn record_import_file_identity( + file: &fs::File, + identities: &mut std::collections::BTreeMap, +) -> Result<(), PortableV2Error> { + let identity = graphforge_filesystem::file_identity(file).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot identify owned import artifact", + ) + })?; + let usage = graphforge_filesystem::file_space_usage(file).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot measure owned import artifact", + ) + })?; + let mut file_id = String::with_capacity(32); + for byte in identity.file_id { + use std::fmt::Write as _; + write!(&mut file_id, "{byte:02x}").expect("writing to String cannot fail"); + } + let key = format!("{:016x}:{file_id}", identity.volume_serial); + identities + .entry(key) + .and_modify(|allocated| *allocated = (*allocated).max(usage.allocated_bytes)) + .or_insert(usage.allocated_bytes); + Ok(()) +} + +fn capture_import_tree( + directory: &graphforge_filesystem::StableDirectory, + identities: &mut std::collections::BTreeMap, + remaining: &mut usize, +) -> Result<(), PortableV2Error> { + let names = directory.child_names_bounded(*remaining).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "completed import staging exceeds identity bound", + ) + })?; + *remaining = remaining.saturating_sub(names.len()); + for name in names { + if let Ok(child) = directory.open_child_directory(&name) { + capture_import_tree(&child, identities, remaining)?; + } else { + let file = directory.open_child_file(&name).map_err(|_| { + PortableV2Error::new( + PortableV2ErrorCode::Io, + "cannot authenticate completed import entry", + ) + })?; + record_import_file_identity(&file, identities)?; + } + } + Ok(()) +} + fn parse_mode(value: &str) -> Result { match value { "exploratory" => Ok(ActivationMode::Exploratory), @@ -1305,6 +1723,10 @@ mod tests { ) .unwrap_err(); assert_eq!(error.code, PortableV2ErrorCode::LimitExceeded, "{name}"); + assert!( + !error.allocation_identity_allocated_bytes.is_empty(), + "{name} must report its durable ownership allocation" + ); assert!(!target.exists(), "{name}"); } } @@ -1397,6 +1819,57 @@ mod tests { } } + #[test] + fn published_import_fails_closed_when_materialization_cleanup_is_not_durable() { + let source_project = tempfile::tempdir().unwrap(); + let source_generation = crate::open_or_initialize_project(source_project.path()).unwrap(); + let package_parent = tempfile::tempdir().unwrap(); + let package = package_parent.path().join("complete.gfproject"); + let limits = crate::PortableV2ExportLimits::default(); + let plan = crate::plan_complete_portable_v2(&source_generation, limits).unwrap(); + crate::export_complete_portable_v2( + &plan, + &package, + crate::PortableV2Output::Expanded, + limits, + &AtomicBool::new(false), + |_| {}, + ) + .unwrap(); + + let target_parent = tempfile::tempdir().unwrap(); + let target = target_parent.path().join("project"); + let transaction = Uuid::new_v4(); + let generation = Uuid::new_v4(); + INJECT_IMPORT_CLEANUP_FAILURE.with(|value| value.set(true)); + let error = import_complete_portable_v2( + &package, + &target, + transaction, + generation, + &supported(), + PortableV2Limits::default(), + None, + ) + .expect_err("cleanup failure must fail closed"); + INJECT_IMPORT_CLEANUP_FAILURE.with(|value| value.set(false)); + assert_eq!( + crate::resolve_project_generation(&target) + .unwrap() + .generation_uuid(), + generation, + "publication may commit, but must not receive a false cleanup receipt" + ); + assert!(!error.allocation_identity_allocated_bytes.is_empty()); + let stage = target_parent + .path() + .join(format!(".project.portable-v2-{}", transaction.hyphenated())); + assert!( + stage.exists(), + "failed cleanup residue must remain attributable" + ); + } + #[test] fn subprocess_crash_import() { let Ok(package) = std::env::var("GRAPHFORGE_PORTABLE_V2_CRASH_PACKAGE") else { diff --git a/crates/graphforge-storage/src/storage_attribution.rs b/crates/graphforge-storage/src/storage_attribution.rs new file mode 100644 index 000000000..d3d646c84 --- /dev/null +++ b/crates/graphforge-storage/src/storage_attribution.rs @@ -0,0 +1,1423 @@ +//! Authenticated, non-enumerating storage attribution for committed projects. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::File; +use std::path::Path; + +use graphforge_core::GfError; +use serde::{Deserialize, Serialize}; + +use crate::{ + GraphConstructionEvidence, GraphFileEntry, GraphFilesParticipant, ResolvedProjectGeneration, +}; + +/// Exhaustive storage categories used by scale qualification evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactCategory { + /// Canonical node topology shards. + TopologyNodes, + /// Canonical edge topology shards and authoritative edge deltas. + TopologyEdges, + /// Node and edge property shards. + Properties, + /// UUID membership and surrogate reverse indexes. + UuidAndSurrogates, + /// Derived adjacency manifests and CSR shards. + Adjacency, + /// Runtime catalogs, generation participants, and compact-manifest nodes. + CatalogAndManifests, + /// Receipt-authenticated construction staging and spill artifacts. + ConstructionStaging, + /// One immutable portable export package. + PortablePackage, + /// The authoritative retained project produced by a clean import. + CleanImportedProject, + /// Unclassified retained graph artifact. Qualification must reject this. + Other, +} + +impl ArtifactCategory { + /// Canonical category inventory, including zero-valued categories. + pub const ALL: [Self; 10] = [ + Self::TopologyNodes, + Self::TopologyEdges, + Self::Properties, + Self::UuidAndSurrogates, + Self::Adjacency, + Self::CatalogAndManifests, + Self::ConstructionStaging, + Self::PortablePackage, + Self::CleanImportedProject, + Self::Other, + ]; +} + +/// Closed lifecycle-phase inventory for application-observed storage I/O. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StorageIoPhase { + /// Chunk append and bounded external merge work. + AppendMerge, + /// Seal-time authentication of staged inputs. + SealAuthentication, + /// Canonical shape consumption and reauthentication. + ShapeConsumeReauthentication, + /// Canonical encoding plus post-write authentication. + EncodeWritePostwriteAuthentication, + /// Publication control preauthentication. + PublicationPreauthentication, + /// Content-addressed installation reads and writes. + CasInstallReadWrite, + /// Workspace hydration and verification. + HydrationVerification, + /// Explicit file and directory synchronization barriers. + FsyncSynchronization, + /// Crash-recovery reauthentication. + RecoveryReauthentication, +} + +impl StorageIoPhase { + /// Complete phase inventory, including phases with zero observations. + pub const ALL: [Self; 9] = [ + Self::AppendMerge, + Self::SealAuthentication, + Self::ShapeConsumeReauthentication, + Self::EncodeWritePostwriteAuthentication, + Self::PublicationPreauthentication, + Self::CasInstallReadWrite, + Self::HydrationVerification, + Self::FsyncSynchronization, + Self::RecoveryReauthentication, + ]; +} + +/// Exact application-I/O totals owned by one lifecycle phase. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PhaseIoTotals { + /// Payload and control bytes returned to the application. + pub read_bytes: u64, + /// Payload and control bytes submitted by the application. + pub write_bytes: u64, + /// Application-observed read calls. + pub read_calls: u64, + /// Application-observed write calls. + pub write_calls: u64, + /// Immutable objects handled by this phase. + pub object_count: u64, + /// Fixed-size authenticated or buffered blocks handled by this phase. + pub block_count: u64, + /// File and directory durability barriers completed by this phase. + pub fsync_calls: u64, +} + +/// Closed, reconciled phase attribution for one construction lifecycle. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ConstructionPhaseAttribution { + /// Every lifecycle phase exactly once, including zero observations. + pub phases: BTreeMap, + /// Exact sum of all phase rows. + pub totals: PhaseIoTotals, +} + +impl ConstructionPhaseAttribution { + /// Derive phase ownership from storage-owned construction counters. + #[must_use] + pub fn from_construction(evidence: &GraphConstructionEvidence) -> Self { + let mut phases: BTreeMap<_, _> = StorageIoPhase::ALL + .into_iter() + .map(|phase| (phase, PhaseIoTotals::default())) + .collect(); + phases.insert( + StorageIoPhase::AppendMerge, + PhaseIoTotals { + read_bytes: evidence.replay_validation_read_bytes, + write_bytes: evidence.write_bytes, + read_calls: evidence.replay_validation_read_operations, + write_calls: evidence.write_operations, + object_count: evidence.parquet_shards, + ..Default::default() + }, + ); + phases.insert( + StorageIoPhase::SealAuthentication, + PhaseIoTotals { + read_bytes: evidence.seal_application_read_bytes, + read_calls: evidence.authentication_read_operations, + ..Default::default() + }, + ); + phases.insert( + StorageIoPhase::ShapeConsumeReauthentication, + shape_phase_totals(evidence), + ); + phases.insert( + StorageIoPhase::EncodeWritePostwriteAuthentication, + PhaseIoTotals { + read_bytes: evidence.encode_application_read_bytes, + write_bytes: evidence.encode_application_write_bytes, + read_calls: evidence.encode_application_read_operations, + write_calls: evidence.encode_application_write_operations, + fsync_calls: evidence.encode_fsync_operations, + ..Default::default() + }, + ); + phases.insert( + StorageIoPhase::PublicationPreauthentication, + PhaseIoTotals { + read_bytes: evidence.publication_application_read_bytes, + read_calls: evidence.publication_application_read_operations, + ..Default::default() + }, + ); + phases.insert( + StorageIoPhase::CasInstallReadWrite, + PhaseIoTotals { + read_bytes: evidence.cas_application_read_bytes, + write_bytes: evidence.cas_application_write_bytes, + read_calls: evidence.cas_application_read_operations, + write_calls: evidence.cas_application_write_operations, + fsync_calls: evidence.cas_fsync_operations, + ..Default::default() + }, + ); + phases.insert( + StorageIoPhase::HydrationVerification, + PhaseIoTotals { + read_bytes: evidence.hydration_application_read_bytes, + write_bytes: evidence.hydration_application_write_bytes, + read_calls: evidence.hydration_application_read_operations, + write_calls: evidence.hydration_application_write_operations, + fsync_calls: evidence.hydration_fsync_operations, + ..Default::default() + }, + ); + phases.insert( + StorageIoPhase::FsyncSynchronization, + PhaseIoTotals { + fsync_calls: evidence + .fsync_operations + .saturating_add(evidence.merge_fsync_operations), + ..Default::default() + }, + ); + phases.insert( + StorageIoPhase::RecoveryReauthentication, + PhaseIoTotals { + read_bytes: evidence.recovery_application_read_bytes, + read_calls: evidence.recovery_application_read_operations, + ..Default::default() + }, + ); + let totals = phases + .values() + .fold(PhaseIoTotals::default(), |mut total, value| { + add_phase_totals_saturating(&mut total, value); + total + }); + Self { phases, totals } + } + + /// Add writer-reported recovery reauthentication completed outside the + /// construction session, such as interrupted portable finalization. + pub fn add_recovery_reauthentication(&mut self, read_bytes: u64, read_calls: u64) { + let recovery = self + .phases + .entry(StorageIoPhase::RecoveryReauthentication) + .or_default(); + recovery.read_bytes = recovery.read_bytes.saturating_add(read_bytes); + recovery.read_calls = recovery.read_calls.saturating_add(read_calls); + self.totals.read_bytes = self.totals.read_bytes.saturating_add(read_bytes); + self.totals.read_calls = self.totals.read_calls.saturating_add(read_calls); + } + + /// Reject missing phases or totals that do not equal the phase sum. + pub fn validate_reconciliation(&self) -> Result<(), GfError> { + if StorageIoPhase::ALL + .iter() + .any(|phase| !self.phases.contains_key(phase)) + { + return Err(validation("storage phase attribution is missing a phase")); + } + let mut total = PhaseIoTotals::default(); + for phase in StorageIoPhase::ALL { + add_phase_totals(&mut total, &self.phases[&phase])?; + } + if total != self.totals { + return Err(validation( + "storage phase attribution totals do not reconcile", + )); + } + Ok(()) + } + + /// Validate the qualification semantics in addition to arithmetic + /// reconciliation. Every lifecycle phase must be present, while a phase + /// that truthfully performed no I/O remains an explicit zero row. Byte and + /// call counters are paired so a synthetic byte-only or call-only row + /// cannot be presented as observed application I/O. + pub fn validate_for_qualification(&self) -> Result<(), GfError> { + self.validate_reconciliation()?; + for phase in StorageIoPhase::ALL { + let totals = &self.phases[&phase]; + if (totals.read_bytes == 0) != (totals.read_calls == 0) { + return Err(validation("phase read bytes and calls disagree")); + } + if (totals.write_bytes == 0) != (totals.write_calls == 0) { + return Err(validation("phase write bytes and calls disagree")); + } + } + Ok(()) + } +} + +fn shape_phase_totals(evidence: &GraphConstructionEvidence) -> PhaseIoTotals { + PhaseIoTotals { + read_bytes: evidence.shape_application_read_bytes, + write_bytes: evidence + .merge_written_bytes + .saturating_add(evidence.parquet_write_bytes), + read_calls: evidence + .shape_input_validation_read_operations + .saturating_add(evidence.merge_read_operations) + .saturating_add(evidence.parquet_read_operations) + .saturating_add(evidence.shaped_output_authentication_operations) + .saturating_add(evidence.parent_catalog_read_operations) + .saturating_add(evidence.retained_probe_block_loads), + write_calls: evidence + .merge_write_operations + .saturating_add(evidence.parquet_write_operations), + block_count: evidence + .merge_read_blocks + .saturating_add(evidence.merge_write_blocks), + ..Default::default() + } +} + +/// Reconciled totals for one artifact category. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactStorageTotals { + /// Logical references in the authenticated inventory. + pub logical_references: u64, + /// Sum of referenced logical bytes; shared objects count per reference. + pub logical_bytes: u64, + /// Distinct retained physical files, deduplicated by native identity. + pub physical_objects: u64, + /// Logical EOF bytes of distinct physical files. + pub physical_logical_bytes: u64, + /// Filesystem-allocated bytes of distinct physical files. + pub allocated_bytes: u64, +} + +/// Authenticated storage attribution for one lifetime-pinned generation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StorageAttributionSnapshot { + /// Selected immutable generation UUID. + pub generation_uuid: uuid::Uuid, + /// SHA-256 of exact authenticated generation manifest bytes. + pub generation_manifest_sha256: [u8; 32], + /// Every category exactly once, including zero totals. + pub categories: BTreeMap, + /// Reconciled logical references across categories. + pub logical_references: u64, + /// Reconciled referenced logical bytes across categories. + pub logical_bytes: u64, + /// Reconciled distinct physical objects across categories. + pub physical_objects: u64, + /// Reconciled distinct-file EOF bytes across categories. + pub physical_logical_bytes: u64, + /// Reconciled distinct-file allocated bytes across categories. + pub allocated_bytes: u64, + /// Distinct native identities and their allocation. This authenticated + /// union is the cross-owner input to lifecycle peak tracking. + #[serde(skip)] + pub physical_identity_allocated_bytes: BTreeMap, +} + +/// Exact native-identity union for the retained project container. +/// +/// This includes `FORMAT`, `CURRENT`, and every authenticated generation still +/// installed in the retained generation namespace. Shared CAS objects are +/// deduplicated by native identity. Cleanup is the only operation allowed to +/// remove a generation from this inventory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectStorageIdentityUnion { + /// Selected generation at the time of capture. + pub selected_generation_uuid: uuid::Uuid, + /// Authenticated generation identities represented in the union. + pub retained_generation_uuids: BTreeSet, + /// Exact native identities and allocated bytes. + pub physical_identity_allocated_bytes: BTreeMap, + /// Reconciled allocation of the identity union. + pub allocated_bytes: u64, +} + +/// One writer-owned change to an authenticated allocation owner. Transitions +/// are replayed in durable operation order so files removed before an API call +/// returns still contribute to the exact full-lifecycle high-water mark. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StorageAllocationTransition { + /// Newly retained native identities and their allocated bytes. + pub installed: BTreeMap, + /// Native identities no longer retained by this owner. + pub removed: BTreeSet, +} + +/// Capture the retained project/container allocation without recursively +/// scanning the project namespace. +/// +/// The bounded generation namespace and each generation's authenticated +/// inventories are the authority. Only the immediate `generations/` directory +/// is enumerated; graph/project payload trees are never recursively scanned. +/// This deliberately includes checkpoint branches and unreachable generations +/// until an explicit cleanup/GC operation removes them. +pub fn capture_project_storage_identity_union( + selected: &ResolvedProjectGeneration, +) -> Result { + const MAX_RETAINED_GENERATIONS: usize = 4_096; + let mut identities = BTreeMap::new(); + for control in ["FORMAT", "CURRENT"] { + let file = File::open(selected.container_root().join(control)).map_err(storage)?; + add_identity_allocation(&mut identities, &file)?; + } + + let generations_root = selected.container_root().join("generations"); + let retained = std::fs::read_dir(&generations_root) + .map_err(storage)? + .map(|entry| { + let entry = entry.map_err(storage)?; + let file_type = entry.file_type().map_err(storage)?; + if !file_type.is_dir() || file_type.is_symlink() { + return Err(validation( + "retained generation namespace contains a non-directory entry", + )); + } + let name = entry + .file_name() + .into_string() + .map_err(|_| validation("retained generation name is not UTF-8"))?; + uuid::Uuid::parse_str(&name) + .map_err(|_| validation("retained generation name is not a UUID")) + }) + .collect::, _>>()?; + if retained.len() > MAX_RETAINED_GENERATIONS { + return Err(validation( + "retained generation namespace exceeds attribution bound", + )); + } + if !retained.contains(&selected.generation_uuid()) { + return Err(validation( + "selected generation is absent from retained namespace", + )); + } + for uuid in &retained { + let generation = crate::resolve_generation_by_uuid(selected.container_root(), *uuid)?; + let snapshot = capture_storage_attribution(&generation)?; + merge_identity_allocations(&mut identities, &snapshot.physical_identity_allocated_bytes)?; + } + let retained_cas = crate::graph_object_store::capture_retained_graph_object_identities( + selected.container_root(), + )?; + merge_identity_allocations(&mut identities, &retained_cas)?; + let allocated_bytes = identities + .values() + .try_fold(0_u64, |total, value| checked_add(total, *value))?; + Ok(ProjectStorageIdentityUnion { + selected_generation_uuid: selected.generation_uuid(), + retained_generation_uuids: retained, + physical_identity_allocated_bytes: identities, + allocated_bytes, + }) +} + +fn add_identity_allocation( + identities: &mut BTreeMap, + file: &File, +) -> Result<(), GfError> { + let identity = graphforge_filesystem::file_identity(file).map_err(storage)?; + let usage = graphforge_filesystem::file_space_usage(file).map_err(storage)?; + let key = native_identity_key(identity.volume_serial, &identity.file_id); + merge_identity_allocations(identities, &BTreeMap::from([(key, usage.allocated_bytes)])) +} + +fn merge_identity_allocations( + target: &mut BTreeMap, + source: &BTreeMap, +) -> Result<(), GfError> { + for (identity, allocated) in source { + if let Some(existing) = target.get(identity) { + if existing != allocated { + return Err(validation("retained identity allocation changed")); + } + } else { + target.insert(identity.clone(), *allocated); + } + } + Ok(()) +} + +/// Exact high-water tracker for simultaneously active authenticated files. +/// Owners are replaced atomically; aliases share one native identity and are +/// counted once until the final owner removes it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StorageAllocationLifecycle { + owners: BTreeMap>, + active: BTreeMap, + current_allocated_bytes: u64, + peak_allocated_bytes: u64, +} + +impl StorageAllocationLifecycle { + /// Replace an owner's exact authenticated identity inventory. + pub fn replace_owner( + &mut self, + owner: impl Into, + identities: &BTreeMap, + ) -> Result<(), GfError> { + let mut candidate = self.clone(); + candidate.replace_owner_inner(owner.into(), identities)?; + *self = candidate; + Ok(()) + } + + fn replace_owner_inner( + &mut self, + owner: String, + identities: &BTreeMap, + ) -> Result<(), GfError> { + self.remove_owner(&owner)?; + let mut installed = BTreeSet::new(); + for (identity, allocated) in identities { + if let Some((existing, references)) = self.active.get_mut(identity) { + if existing != allocated { + return Err(validation("active identity allocation changed")); + } + *references = checked_add(*references, 1)?; + } else { + self.current_allocated_bytes = + checked_add(self.current_allocated_bytes, *allocated)?; + self.active.insert(identity.clone(), (*allocated, 1)); + } + installed.insert(identity.clone()); + self.peak_allocated_bytes = self.peak_allocated_bytes.max(self.current_allocated_bytes); + } + self.owners.insert(owner, installed); + Ok(()) + } + + /// Replace an owner from a generation-bound storage snapshot. + pub fn replace_snapshot_owner( + &mut self, + owner: impl Into, + snapshot: &StorageAttributionSnapshot, + ) -> Result<(), GfError> { + snapshot.validate_reconciliation()?; + self.replace_owner(owner, &snapshot.physical_identity_allocated_bytes) + } + + /// Apply one writer-owned install/remove transition without reconstructing + /// an operation's historical state from its final filesystem layout. + pub fn apply_owner_transition( + &mut self, + owner: impl Into, + transition: &StorageAllocationTransition, + ) -> Result<(), GfError> { + let mut candidate = self.clone(); + candidate.apply_owner_transition_inner(owner.into(), transition)?; + *self = candidate; + Ok(()) + } + + fn apply_owner_transition_inner( + &mut self, + owner: String, + transition: &StorageAllocationTransition, + ) -> Result<(), GfError> { + if transition + .installed + .keys() + .any(|identity| transition.removed.contains(identity)) + { + return Err(validation( + "allocation transition installs and removes one identity", + )); + } + let owner_identities = self.owners.entry(owner).or_default(); + for identity in &transition.removed { + if !owner_identities.remove(identity) { + return Err(validation( + "allocation transition removes an unowned identity", + )); + } + let (allocated, references) = self + .active + .get(identity) + .copied() + .ok_or_else(|| validation("active transition identity is absent"))?; + if references == 1 { + self.active.remove(identity); + self.current_allocated_bytes = self + .current_allocated_bytes + .checked_sub(allocated) + .ok_or_else(|| validation("active allocation underflow"))?; + } else { + self.active + .insert(identity.clone(), (allocated, references - 1)); + } + } + for (identity, allocated) in &transition.installed { + if !owner_identities.insert(identity.clone()) { + return Err(validation( + "allocation transition installs an owned identity", + )); + } + if let Some((existing, references)) = self.active.get_mut(identity) { + if existing != allocated { + return Err(validation("active identity allocation changed")); + } + *references = checked_add(*references, 1)?; + } else { + self.current_allocated_bytes = + checked_add(self.current_allocated_bytes, *allocated)?; + self.active.insert(identity.clone(), (*allocated, 1)); + } + self.peak_allocated_bytes = self.peak_allocated_bytes.max(self.current_allocated_bytes); + } + Ok(()) + } + + /// Remove an owner and decrement every exact identity reference. + pub fn remove_owner(&mut self, owner: &str) -> Result<(), GfError> { + let Some(identities) = self.owners.remove(owner) else { + return Ok(()); + }; + for identity in identities { + let (allocated, references) = self + .active + .get(&identity) + .copied() + .ok_or_else(|| validation("active identity owner is absent"))?; + if references == 1 { + self.active.remove(&identity); + self.current_allocated_bytes = self + .current_allocated_bytes + .checked_sub(allocated) + .ok_or_else(|| validation("active allocation underflow"))?; + } else { + self.active.insert(identity, (allocated, references - 1)); + } + } + Ok(()) + } + + /// Current exact identity-union allocation. + #[must_use] + pub const fn current_allocated_bytes(&self) -> u64 { + self.current_allocated_bytes + } + + /// Exact high-water allocation observed after every owner transition. + #[must_use] + pub const fn peak_allocated_bytes(&self) -> u64 { + self.peak_allocated_bytes + } +} + +impl StorageAttributionSnapshot { + /// Whether every retained graph artifact was assigned a qualifying category. + #[must_use] + pub fn is_fully_classified(&self) -> bool { + self.categories + .get(&ArtifactCategory::Other) + .is_none_or(|totals| totals.logical_references == 0 && totals.physical_objects == 0) + } + + /// Recompute and validate snapshot totals. + pub fn validate_reconciliation(&self) -> Result<(), GfError> { + if ArtifactCategory::ALL + .iter() + .any(|category| !self.categories.contains_key(category)) + { + return Err(validation("storage attribution is missing a category")); + } + let mut total = ArtifactStorageTotals::default(); + for category in ArtifactCategory::ALL { + let value = &self.categories[&category]; + add_totals(&mut total, value)?; + } + if total.logical_references != self.logical_references + || total.logical_bytes != self.logical_bytes + || total.physical_objects != self.physical_objects + || total.physical_logical_bytes != self.physical_logical_bytes + || total.allocated_bytes != self.allocated_bytes + { + return Err(validation("storage attribution totals do not reconcile")); + } + let identity_allocated = self + .physical_identity_allocated_bytes + .values() + .try_fold(0_u64, |total, value| checked_add(total, *value))?; + if identity_allocated != self.allocated_bytes + || self.physical_identity_allocated_bytes.len() as u64 != self.physical_objects + { + return Err(validation( + "storage attribution identity union does not reconcile", + )); + } + Ok(()) + } + + /// Validate the stricter scale-qualification contract. + /// + /// Qualification is fail-closed when any retained graph artifact remains + /// in [`ArtifactCategory::Other`]. + pub fn validate_for_qualification(&self) -> Result<(), GfError> { + self.validate_reconciliation()?; + if !self.is_fully_classified() { + return Err(validation( + "storage attribution contains unclassified retained artifacts", + )); + } + Ok(()) + } +} + +/// Classify one authenticated graph inventory path. +#[must_use] +pub fn classify_graph_artifact(relative_path: &str) -> ArtifactCategory { + let path = Path::new(relative_path); + let mut components = path.components().filter_map(|component| match component { + std::path::Component::Normal(value) => value.to_str(), + _ => None, + }); + match (components.next(), components.next()) { + (Some("topology"), Some("nodes" | "nodes.parquet")) => ArtifactCategory::TopologyNodes, + (Some("topology"), Some("edges" | "uuid-membership")) => { + if relative_path.starts_with("topology/uuid-membership/") { + ArtifactCategory::UuidAndSurrogates + } else { + ArtifactCategory::TopologyEdges + } + } + (Some("topology"), Some("surrogate_tails.parquet")) => ArtifactCategory::UuidAndSurrogates, + (Some("topology"), Some("runtime_catalog.parquet" | "generation.json")) => { + ArtifactCategory::CatalogAndManifests + } + (Some("deltas"), _) => ArtifactCategory::TopologyEdges, + (Some("properties" | "edge_properties"), _) => ArtifactCategory::Properties, + (Some("indexes" | "index"), Some("adjacency")) => ArtifactCategory::Adjacency, + (Some("indexes" | "index"), Some(name)) + if name.contains("uuid") || name.contains("surrogate") => + { + ArtifactCategory::UuidAndSurrogates + } + (Some(name), _) if name.starts_with("runtime_catalog") => { + ArtifactCategory::CatalogAndManifests + } + _ => ArtifactCategory::Other, + } +} + +/// Capture attribution from a pinned generation and its authenticated compact +/// inventory. No project directory is recursively enumerated. +pub fn capture_storage_attribution( + generation: &ResolvedProjectGeneration, +) -> Result { + let mut accumulator = Accumulator::new(generation); + let generation_root = + graphforge_filesystem::StableDirectory::open(generation.generation_root()) + .map_err(storage)?; + let generation_manifest = generation_root + .open_child_file(std::ffi::OsStr::new("manifest.json")) + .map_err(storage)?; + let generation_manifest_usage = + graphforge_filesystem::file_space_usage(&generation_manifest).map_err(storage)?; + accumulator.add_logical( + ArtifactCategory::CatalogAndManifests, + generation_manifest_usage.logical_bytes, + )?; + accumulator.add_physical( + ArtifactCategory::CatalogAndManifests, + &generation_manifest, + generation_manifest_usage.logical_bytes, + )?; + for descriptor in generation.participant_descriptors()? { + let Some(snapshot) = generation + .participant_snapshot(&descriptor.capability_id, &descriptor.record_family_id)? + else { + return Err(validation("declared participant disappeared")); + }; + let path = + generation.participant_path(&descriptor.capability_id, &descriptor.record_family_id)?; + let file = File::open(&path).map_err(storage)?; + accumulator.add_physical( + ArtifactCategory::CatalogAndManifests, + &file, + u64::try_from(snapshot.bytes.len()).map_err(|_| validation("participant too large"))?, + )?; + accumulator.add_logical( + ArtifactCategory::CatalogAndManifests, + u64::try_from(snapshot.bytes.len()).map_err(|_| validation("participant too large"))?, + )?; + } + + match generation.declared_graph_files_participant()? { + Some(GraphFilesParticipant::V2(root)) => { + let lease = + crate::graph_object_store::begin_graph_object_read(generation.container_root())?; + let mut manifest_objects = BTreeSet::new(); + let (entries, _) = crate::resolve_graph_manifest( + &root, + crate::GraphManifestLimits::default(), + |digest| { + let bytes = crate::read_graph_object_by_digest( + generation.container_root(), + digest, + 64 * 1024 * 1024, + )?; + manifest_objects.insert(( + digest.to_owned(), + u64::try_from(bytes.len()) + .map_err(|_| validation("graph manifest object too large"))?, + )); + Ok(bytes) + }, + )?; + for (digest, length) in manifest_objects { + let object = lease.open(&digest, length)?; + accumulator.add_physical( + ArtifactCategory::CatalogAndManifests, + object.as_ref(), + length, + )?; + } + for entry in entries { + add_compact_entry(&mut accumulator, &lease, &entry)?; + } + } + Some(GraphFilesParticipant::V1(inventory)) => { + crate::verify_graph_tree(&generation.graph_tree_root(), &inventory)?; + let graph_root = generation.graph_tree_root(); + for entry in inventory.files { + let category = classify_graph_artifact(&entry.relative_path); + accumulator.add_logical(category, entry.byte_length)?; + let file = open_inventory_file(&graph_root, &entry.relative_path)?; + accumulator.add_physical(category, &file, entry.byte_length)?; + } + } + None => {} + } + accumulator.finish() +} + +fn open_inventory_file(root: &Path, relative: &str) -> Result { + let components = Path::new(relative) + .components() + .map(|component| match component { + std::path::Component::Normal(value) => Ok(value.to_owned()), + _ => Err(validation("graph inventory path is not normalized")), + }) + .collect::, _>>()?; + let (file_name, directories) = components + .split_last() + .ok_or_else(|| validation("graph inventory path is empty"))?; + let mut directory = graphforge_filesystem::StableDirectory::open(root).map_err(storage)?; + for name in directories { + directory = directory.open_child_directory(name).map_err(storage)?; + } + directory.open_child_file(file_name).map_err(storage) +} + +fn add_compact_entry( + accumulator: &mut Accumulator, + lease: &crate::graph_object_store::GraphObjectReadLease, + entry: &GraphFileEntry, +) -> Result<(), GfError> { + let category = classify_graph_artifact(&entry.relative_path); + accumulator.add_logical(category, entry.byte_length)?; + let object = lease.open(&entry.content_sha256, entry.byte_length)?; + accumulator.add_physical(category, object.as_ref(), entry.byte_length) +} + +struct Accumulator { + generation_uuid: uuid::Uuid, + generation_manifest_sha256: [u8; 32], + categories: BTreeMap, + physical_seen: BTreeSet<(u64, [u8; 16])>, + physical_identity_allocated_bytes: BTreeMap, +} + +impl Accumulator { + fn new(generation: &ResolvedProjectGeneration) -> Self { + Self { + generation_uuid: generation.generation_uuid(), + generation_manifest_sha256: generation.manifest_sha256(), + categories: ArtifactCategory::ALL + .into_iter() + .map(|category| (category, ArtifactStorageTotals::default())) + .collect(), + physical_seen: BTreeSet::new(), + physical_identity_allocated_bytes: BTreeMap::new(), + } + } + + fn add_logical(&mut self, category: ArtifactCategory, bytes: u64) -> Result<(), GfError> { + let totals = self + .categories + .get_mut(&category) + .expect("complete categories"); + totals.logical_references = checked_add(totals.logical_references, 1)?; + totals.logical_bytes = checked_add(totals.logical_bytes, bytes)?; + Ok(()) + } + + fn add_physical( + &mut self, + category: ArtifactCategory, + file: &File, + expected_logical_bytes: u64, + ) -> Result<(), GfError> { + let identity = graphforge_filesystem::file_identity(file).map_err(storage)?; + let usage = graphforge_filesystem::file_space_usage(file).map_err(storage)?; + if usage.logical_bytes != expected_logical_bytes { + return Err(validation( + "authenticated artifact length changed during attribution", + )); + } + if self + .physical_seen + .insert((identity.volume_serial, identity.file_id)) + { + self.physical_identity_allocated_bytes.insert( + native_identity_key(identity.volume_serial, &identity.file_id), + usage.allocated_bytes, + ); + let totals = self + .categories + .get_mut(&category) + .expect("complete categories"); + totals.physical_objects = checked_add(totals.physical_objects, 1)?; + totals.physical_logical_bytes = + checked_add(totals.physical_logical_bytes, usage.logical_bytes)?; + totals.allocated_bytes = checked_add(totals.allocated_bytes, usage.allocated_bytes)?; + } + Ok(()) + } + + fn finish(self) -> Result { + let mut total = ArtifactStorageTotals::default(); + for value in self.categories.values() { + add_totals(&mut total, value)?; + } + let snapshot = StorageAttributionSnapshot { + generation_uuid: self.generation_uuid, + generation_manifest_sha256: self.generation_manifest_sha256, + categories: self.categories, + logical_references: total.logical_references, + logical_bytes: total.logical_bytes, + physical_objects: total.physical_objects, + physical_logical_bytes: total.physical_logical_bytes, + allocated_bytes: total.allocated_bytes, + physical_identity_allocated_bytes: self.physical_identity_allocated_bytes, + }; + snapshot.validate_reconciliation()?; + Ok(snapshot) + } +} + +fn add_totals( + target: &mut ArtifactStorageTotals, + value: &ArtifactStorageTotals, +) -> Result<(), GfError> { + target.logical_references = checked_add(target.logical_references, value.logical_references)?; + target.logical_bytes = checked_add(target.logical_bytes, value.logical_bytes)?; + target.physical_objects = checked_add(target.physical_objects, value.physical_objects)?; + target.physical_logical_bytes = + checked_add(target.physical_logical_bytes, value.physical_logical_bytes)?; + target.allocated_bytes = checked_add(target.allocated_bytes, value.allocated_bytes)?; + Ok(()) +} + +fn add_phase_totals(target: &mut PhaseIoTotals, value: &PhaseIoTotals) -> Result<(), GfError> { + target.read_bytes = checked_add(target.read_bytes, value.read_bytes)?; + target.write_bytes = checked_add(target.write_bytes, value.write_bytes)?; + target.read_calls = checked_add(target.read_calls, value.read_calls)?; + target.write_calls = checked_add(target.write_calls, value.write_calls)?; + target.object_count = checked_add(target.object_count, value.object_count)?; + target.block_count = checked_add(target.block_count, value.block_count)?; + target.fsync_calls = checked_add(target.fsync_calls, value.fsync_calls)?; + Ok(()) +} + +fn add_phase_totals_saturating(target: &mut PhaseIoTotals, value: &PhaseIoTotals) { + target.read_bytes = target.read_bytes.saturating_add(value.read_bytes); + target.write_bytes = target.write_bytes.saturating_add(value.write_bytes); + target.read_calls = target.read_calls.saturating_add(value.read_calls); + target.write_calls = target.write_calls.saturating_add(value.write_calls); + target.object_count = target.object_count.saturating_add(value.object_count); + target.block_count = target.block_count.saturating_add(value.block_count); + target.fsync_calls = target.fsync_calls.saturating_add(value.fsync_calls); +} + +fn checked_add(left: u64, right: u64) -> Result { + left.checked_add(right) + .ok_or_else(|| validation("storage attribution counter overflow")) +} + +fn native_identity_key(volume_serial: u64, file_id: &[u8; 16]) -> String { + use std::fmt::Write as _; + let mut value = format!("{volume_serial:016x}:"); + for byte in file_id { + write!(&mut value, "{byte:02x}").expect("writing to String cannot fail"); + } + value +} + +fn validation(message: impl Into) -> GfError { + GfError::Validation(message.into()) +} + +fn storage(error: impl std::fmt::Display) -> GfError { + GfError::Storage(error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + + fn publish_compact_fixture( + project: &Path, + workspace: &Path, + ) -> crate::ResolvedProjectGeneration { + let (_, graph) = crate::capture_graph_files(workspace).unwrap(); + let mut participants = crate::empty_workspace_participants().unwrap(); + participants.insert(0, graph); + let request = crate::ProjectGenerationRequest { + transaction_uuid: uuid::Uuid::now_v7(), + generation_uuid: uuid::Uuid::now_v7(), + capabilities: vec![ + crate::ProjectCapability { + capability_id: crate::GRAPH_CAPABILITY_ID.into(), + capability_version: crate::GRAPH_CAPABILITY_VERSION, + }, + crate::ProjectCapability { + capability_id: "workspace".into(), + capability_version: 1, + }, + ], + participants, + }; + let crate::ProjectStageOutcome::Staged(staged) = + crate::stage_project_generation_with_graph_tree(project, &request, Some(workspace)) + .unwrap() + else { + panic!("fresh compact fixture unexpectedly replayed"); + }; + staged + .validate(|_| Ok(()), |_, _| Ok(())) + .unwrap() + .publish() + .unwrap(); + crate::resolve_project_generation(project).unwrap() + } + + #[test] + fn classifier_is_exhaustive_and_specific() { + assert_eq!( + classify_graph_artifact("topology/nodes/1.parquet"), + ArtifactCategory::TopologyNodes + ); + assert_eq!( + classify_graph_artifact("topology/edges/KNOWS/1.parquet"), + ArtifactCategory::TopologyEdges + ); + assert_eq!( + classify_graph_artifact("properties/Person/1.parquet"), + ArtifactCategory::Properties + ); + assert_eq!( + classify_graph_artifact("topology/uuid-membership/manifest.json"), + ArtifactCategory::UuidAndSurrogates + ); + assert_eq!( + classify_graph_artifact("indexes/adjacency/_all.out.csr"), + ArtifactCategory::Adjacency + ); + assert_eq!( + classify_graph_artifact("runtime_catalog.parquet"), + ArtifactCategory::CatalogAndManifests + ); + assert_eq!( + classify_graph_artifact("topology/runtime_catalog.parquet"), + ArtifactCategory::CatalogAndManifests + ); + assert_eq!( + classify_graph_artifact("topology/generation.json"), + ArtifactCategory::CatalogAndManifests + ); + assert_eq!( + classify_graph_artifact("topology/surrogate_tails.parquet"), + ArtifactCategory::UuidAndSurrogates + ); + assert_eq!( + classify_graph_artifact("unknown.bin"), + ArtifactCategory::Other + ); + } + + #[test] + fn reconciliation_rejects_mismatch_and_missing_category() { + let mut categories: BTreeMap<_, _> = ArtifactCategory::ALL + .into_iter() + .map(|category| (category, ArtifactStorageTotals::default())) + .collect(); + categories + .get_mut(&ArtifactCategory::TopologyNodes) + .unwrap() + .logical_bytes = 7; + let mut snapshot = StorageAttributionSnapshot { + generation_uuid: uuid::Uuid::nil(), + generation_manifest_sha256: [0; 32], + categories, + logical_references: 0, + logical_bytes: 7, + physical_objects: 0, + physical_logical_bytes: 0, + allocated_bytes: 0, + physical_identity_allocated_bytes: BTreeMap::new(), + }; + snapshot.validate_reconciliation().unwrap(); + snapshot.logical_bytes = 8; + assert!(snapshot.validate_reconciliation().is_err()); + snapshot.categories.remove(&ArtifactCategory::Other); + assert!(snapshot.validate_reconciliation().is_err()); + } + + #[test] + fn qualification_rejects_other_artifacts() { + let mut categories: BTreeMap<_, _> = ArtifactCategory::ALL + .into_iter() + .map(|category| (category, ArtifactStorageTotals::default())) + .collect(); + categories + .get_mut(&ArtifactCategory::Other) + .unwrap() + .logical_references = 1; + let snapshot = StorageAttributionSnapshot { + generation_uuid: uuid::Uuid::nil(), + generation_manifest_sha256: [0; 32], + categories, + logical_references: 1, + logical_bytes: 0, + physical_objects: 0, + physical_logical_bytes: 0, + allocated_bytes: 0, + physical_identity_allocated_bytes: BTreeMap::new(), + }; + assert!(snapshot.validate_reconciliation().is_ok()); + assert!(snapshot.validate_for_qualification().is_err()); + } + + #[test] + fn lifecycle_union_deduplicates_identities_and_decrements_owners() { + let mut lifecycle = StorageAllocationLifecycle::default(); + let first = BTreeMap::from([("dev:a".to_owned(), 4096), ("dev:b".to_owned(), 8192)]); + let alias = BTreeMap::from([("dev:b".to_owned(), 8192), ("dev:c".to_owned(), 4096)]); + lifecycle.replace_owner("source", &first).unwrap(); + assert_eq!(lifecycle.current_allocated_bytes(), 12_288); + lifecycle.replace_owner("import", &alias).unwrap(); + assert_eq!(lifecycle.current_allocated_bytes(), 16_384); + assert_eq!(lifecycle.peak_allocated_bytes(), 16_384); + lifecycle.remove_owner("source").unwrap(); + assert_eq!(lifecycle.current_allocated_bytes(), 12_288); + lifecycle.remove_owner("import").unwrap(); + assert_eq!(lifecycle.current_allocated_bytes(), 0); + assert_eq!(lifecycle.peak_allocated_bytes(), 16_384); + } + + #[test] + fn project_union_keeps_noncurrent_generations_and_deduplicates_shared_cas_identity() { + let project = tempfile::tempdir().unwrap(); + let initial = crate::open_or_initialize_project(project.path()).unwrap(); + let workspace = tempfile::tempdir().unwrap(); + let topology = workspace.path().join("topology"); + std::fs::create_dir_all(&topology).unwrap(); + std::fs::write(topology.join("nodes.parquet"), b"shared compact payload").unwrap(); + + let first = publish_compact_fixture(project.path(), workspace.path()); + let first_snapshot = capture_storage_attribution(&first).unwrap(); + let current = publish_compact_fixture(project.path(), workspace.path()); + let current_snapshot = capture_storage_attribution(¤t).unwrap(); + let union = capture_project_storage_identity_union(¤t).unwrap(); + + assert_ne!(first.generation_uuid(), current.generation_uuid()); + assert!( + union + .retained_generation_uuids + .contains(&initial.generation_uuid()) + ); + assert!( + union + .retained_generation_uuids + .contains(&first.generation_uuid()) + ); + assert!( + union + .retained_generation_uuids + .contains(¤t.generation_uuid()) + ); + for identity in first_snapshot + .physical_identity_allocated_bytes + .keys() + .chain(current_snapshot.physical_identity_allocated_bytes.keys()) + { + assert!( + union + .physical_identity_allocated_bytes + .contains_key(identity) + ); + } + let mut repeated_reference = first_snapshot.physical_identity_allocated_bytes.clone(); + merge_identity_allocations( + &mut repeated_reference, + &first_snapshot.physical_identity_allocated_bytes, + ) + .unwrap(); + assert_eq!( + repeated_reference, first_snapshot.physical_identity_allocated_bytes, + "a shared physical identity must remain one union member" + ); + let mut deduplicated = first_snapshot.physical_identity_allocated_bytes.clone(); + merge_identity_allocations( + &mut deduplicated, + ¤t_snapshot.physical_identity_allocated_bytes, + ) + .unwrap(); + assert_eq!( + union.allocated_bytes, + union + .physical_identity_allocated_bytes + .values() + .copied() + .sum::() + ); + } + + #[test] + fn project_union_retains_unreferenced_cas_identity_until_explicit_gc_receipt() { + let project = tempfile::tempdir().unwrap(); + let generation = crate::open_or_initialize_project(project.path()).unwrap(); + let (digest, installed) = crate::graph_object_store::install_graph_object_bytes( + project.path(), + b"unreferenced retained CAS payload", + ) + .unwrap(); + assert!(installed.bytes_installed > 0); + let object = File::open( + crate::graph_object_store::graph_object_path(project.path(), &digest).unwrap(), + ) + .unwrap(); + let identity = graphforge_filesystem::file_identity(&object).unwrap(); + let key = native_identity_key(identity.volume_serial, &identity.file_id); + + let before = capture_project_storage_identity_union(&generation).unwrap(); + assert!( + before.physical_identity_allocated_bytes.contains_key(&key), + "sealed CAS remains retained even when no generation references it" + ); + let gc = crate::graph_object_store::gc_graph_objects( + project.path(), + &[], + crate::GraphManifestLimits::default(), + ) + .unwrap(); + assert_eq!(gc.objects_removed, 1); + assert!(gc.bytes_removed > 0); + assert_eq!( + gc.removed_identity_allocated_bytes.get(&key), + before.physical_identity_allocated_bytes.get(&key) + ); + let reopened = crate::resolve_project_generation(project.path()).unwrap(); + let after = capture_project_storage_identity_union(&reopened).unwrap(); + assert!(!after.physical_identity_allocated_bytes.contains_key(&key)); + assert!(after.allocated_bytes < before.allocated_bytes); + } + + #[test] + fn lifecycle_union_rejects_identity_allocation_disagreement() { + let mut lifecycle = StorageAllocationLifecycle::default(); + lifecycle + .replace_owner("first", &BTreeMap::from([("dev:a".to_owned(), 4096)])) + .unwrap(); + assert!( + lifecycle + .replace_owner("alias", &BTreeMap::from([("dev:a".to_owned(), 8192)])) + .is_err() + ); + } + + #[test] + fn lifecycle_transition_preserves_removed_intra_operation_peak() { + let mut lifecycle = StorageAllocationLifecycle::default(); + lifecycle + .apply_owner_transition( + "construction", + &StorageAllocationTransition { + installed: BTreeMap::from([ + ("dev:staging".to_owned(), 4096), + ("dev:merge".to_owned(), 8192), + ]), + removed: BTreeSet::new(), + }, + ) + .unwrap(); + lifecycle + .apply_owner_transition( + "construction", + &StorageAllocationTransition { + installed: BTreeMap::from([("dev:encoded".to_owned(), 16_384)]), + removed: BTreeSet::new(), + }, + ) + .unwrap(); + lifecycle + .apply_owner_transition( + "construction", + &StorageAllocationTransition { + installed: BTreeMap::new(), + removed: BTreeSet::from(["dev:staging".to_owned(), "dev:merge".to_owned()]), + }, + ) + .unwrap(); + assert_eq!(lifecycle.current_allocated_bytes(), 16_384); + assert_eq!(lifecycle.peak_allocated_bytes(), 28_672); + } + + #[test] + fn construction_phase_inventory_reconciles_and_rejects_omission() { + let evidence = GraphConstructionEvidence { + seal_application_read_bytes: 11, + shape_application_read_bytes: 13, + shape_input_validation_read_operations: 1, + merge_read_operations: 2, + parquet_read_operations: 3, + shaped_output_authentication_operations: 4, + parent_catalog_read_operations: 5, + retained_probe_block_loads: 6, + merge_written_bytes: 5, + merge_write_operations: 2, + parquet_write_bytes: 7, + parquet_write_operations: 1, + encode_application_read_bytes: 17, + encode_application_read_operations: 2, + encode_application_write_bytes: 31, + encode_application_write_operations: 4, + encode_fsync_operations: 9, + publication_application_read_bytes: 19, + publication_application_read_operations: 2, + cas_application_read_bytes: 23, + cas_application_read_operations: 3, + cas_application_write_bytes: 43, + cas_application_write_operations: 4, + cas_fsync_operations: 5, + hydration_application_read_bytes: 29, + hydration_application_read_operations: 6, + hydration_application_write_bytes: 47, + hydration_application_write_operations: 7, + hydration_fsync_operations: 8, + recovery_application_read_bytes: 41, + recovery_application_read_operations: 2, + canonical_output_bytes: 31, + write_bytes: 37, + write_operations: 3, + authentication_read_operations: 5, + merge_fsync_operations: 7, + ..Default::default() + }; + let mut attribution = ConstructionPhaseAttribution::from_construction(&evidence); + attribution.validate_reconciliation().unwrap(); + attribution.validate_for_qualification().unwrap(); + assert_eq!(attribution.phases.len(), StorageIoPhase::ALL.len()); + assert_eq!( + attribution.phases[&StorageIoPhase::ShapeConsumeReauthentication], + PhaseIoTotals { + read_bytes: 13, + write_bytes: 12, + read_calls: 21, + write_calls: 3, + ..Default::default() + } + ); + assert_eq!(attribution.totals.read_bytes, 153); + assert_eq!( + attribution.phases[&StorageIoPhase::RecoveryReauthentication].read_calls, + 2 + ); + attribution.add_recovery_reauthentication(9, 1); + attribution.validate_for_qualification().unwrap(); + assert_eq!( + attribution.phases[&StorageIoPhase::RecoveryReauthentication].read_bytes, + 50 + ); + assert_eq!(attribution.totals.read_bytes, 162); + assert_eq!(attribution.totals.write_bytes, 170); + assert_eq!(attribution.totals.read_calls, 42); + assert_eq!(attribution.totals.write_calls, 21); + assert_eq!(attribution.totals.fsync_calls, 29); + attribution + .phases + .remove(&StorageIoPhase::RecoveryReauthentication); + assert!(attribution.validate_reconciliation().is_err()); + } + + #[test] + fn construction_phase_inventory_rejects_double_counted_total() { + let mut attribution = + ConstructionPhaseAttribution::from_construction(&GraphConstructionEvidence::default()); + attribution.totals.read_bytes = 1; + assert!(attribution.validate_reconciliation().is_err()); + } + + #[test] + fn qualification_preserves_truthful_zero_io_phase_rows() { + let attribution = + ConstructionPhaseAttribution::from_construction(&GraphConstructionEvidence::default()); + attribution.validate_for_qualification().unwrap(); + assert_eq!(attribution.phases.len(), StorageIoPhase::ALL.len()); + assert!( + attribution + .phases + .values() + .all(|totals| totals == &PhaseIoTotals::default()) + ); + } + + #[test] + fn one_physical_identity_is_counted_once_for_shared_references() { + let project = tempfile::tempdir().unwrap(); + let generation = crate::open_or_initialize_ephemeral_project(project.path()).unwrap(); + let artifact = tempfile::NamedTempFile::new().unwrap(); + artifact.as_file().write_all(b"shared").unwrap(); + artifact.as_file().sync_all().unwrap(); + let mut accumulator = Accumulator::new(&generation); + accumulator + .add_logical(ArtifactCategory::TopologyNodes, 6) + .unwrap(); + accumulator + .add_logical(ArtifactCategory::Properties, 6) + .unwrap(); + accumulator + .add_physical(ArtifactCategory::TopologyNodes, artifact.as_file(), 6) + .unwrap(); + accumulator + .add_physical(ArtifactCategory::Properties, artifact.as_file(), 6) + .unwrap(); + let snapshot = accumulator.finish().unwrap(); + assert_eq!(snapshot.logical_references, 2); + assert_eq!(snapshot.logical_bytes, 12); + assert_eq!(snapshot.physical_objects, 1); + assert_eq!(snapshot.physical_logical_bytes, 6); + assert!(snapshot.allocated_bytes >= 6); + } +} diff --git a/docs/development/evidence/g500-certification.schema.json b/docs/development/evidence/g500-certification.schema.json index 0f9ced2fb..881a16824 100644 --- a/docs/development/evidence/g500-certification.schema.json +++ b/docs/development/evidence/g500-certification.schema.json @@ -4,12 +4,12 @@ "title": "GraphForge billion-live-edge certification evidence", "type": "object", "additionalProperties": false, - "required": ["schema", "git_sha", "profile_sha256", "run", "host", "tools", "counts", "identities", "package", "authority", "equivalence", "phases", "envelope", "result", "first_failure"], + "required": ["schema", "git_sha", "profile_sha256", "run", "host", "tools", "counts", "identities", "package", "authority", "equivalence", "storage_attribution", "phases", "envelope", "result", "first_failure"], "properties": { "schema": { "const": "graphforge-billion-edge-certification-evidence/1" }, "git_sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, "profile_sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, - "run": { "type": "object", "additionalProperties": false, "required": ["command", "scale", "edgefactor", "seed", "directionality", "self_loops", "duplicates"], "properties": { "command": { "const": "cargo test -p graphforge-api --release --test scale_g500_ladder certification_target_live_full_lifecycle_evidence -- --ignored --exact --nocapture --test-threads=1" }, "scale": { "const": 26 }, "edgefactor": { "const": 16 }, "seed": { "const": 1 }, "directionality": { "const": "undirected" }, "self_loops": { "const": "drop" }, "duplicates": { "const": "drop" } } }, + "run": { "type": "object", "additionalProperties": false, "required": ["command", "scale", "edgefactor", "seed", "directionality", "self_loops", "duplicates"], "properties": { "command": { "const": "cargo test -p graphforge-api --release --test scale_g500_ladder certification_target_live_full_lifecycle_evidence -- --ignored --exact --nocapture --test-threads=1" }, "scale": { "enum": [20, 22, 24, 26] }, "edgefactor": { "const": 16 }, "seed": { "const": 1 }, "directionality": { "const": "undirected" }, "self_loops": { "const": "drop" }, "duplicates": { "const": "drop" } } }, "host": { "type": "object", "additionalProperties": false, "required": ["provider", "region", "sku", "os_image", "os", "kernel", "filesystem", "memory_bytes", "nvme_bytes"], @@ -21,8 +21,8 @@ "os": { "type": "string", "pattern": "^Linux", "description": "Observed Linux OS family from the provisioned host." }, "kernel": { "type": "string", "minLength": 1, "maxLength": 128 }, "filesystem": { "enum": ["ext4", "xfs", "btrfs"], "description": "Filesystem of the Rust process temp workspace; provisioning proves it is the same local-NVMe device as RUNNER_TEMP." }, - "memory_bytes": { "type": "integer", "minimum": 137438953472 }, - "nvme_bytes": { "type": "integer", "minimum": 1099511627776 } + "memory_bytes": { "type": "integer", "minimum": 1, "maximum": 137438953472 }, + "nvme_bytes": { "type": "integer", "minimum": 1 } } }, "tools": { "type": "object", "additionalProperties": { "type": "string", "maxLength": 128 } }, @@ -30,34 +30,86 @@ "type": "object", "additionalProperties": false, "required": ["raw_attempts", "self_loops_rejected", "duplicates_rejected", "live_unique_edges", "source_nodes", "source_edges", "imported_nodes", "imported_edges"], "properties": { - "raw_attempts": { "type": "integer", "minimum": 1000000000 }, + "raw_attempts": { "type": "integer", "minimum": 1 }, "self_loops_rejected": { "type": "integer", "minimum": 0 }, "duplicates_rejected": { "type": "integer", "minimum": 0 }, - "live_unique_edges": { "type": "integer", "minimum": 1000000000 }, - "source_nodes": { "const": 67108864 }, "source_edges": { "type": "integer", "minimum": 1000000000 }, - "imported_nodes": { "const": 67108864 }, "imported_edges": { "type": "integer", "minimum": 1000000000 } + "live_unique_edges": { "type": "integer", "minimum": 1 }, + "source_nodes": { "type": "integer", "minimum": 1 }, "source_edges": { "type": "integer", "minimum": 1 }, + "imported_nodes": { "type": "integer", "minimum": 1 }, "imported_edges": { "type": "integer", "minimum": 1 } } }, "identities": { "type": "object", "additionalProperties": false, - "required": ["source_generation", "package", "transport", "imported_generation"], + "required": ["source_export_generation_authenticated", "import_receipt_reopen_authenticated", "source_import_generations_distinct", "package", "transport"], "properties": { - "source_generation": { "$ref": "#/$defs/nonSecretIdentity" }, + "source_export_generation_authenticated": { "const": true }, + "import_receipt_reopen_authenticated": { "const": true }, + "source_import_generations_distinct": { "const": true }, "package": { "$ref": "#/$defs/sha256" }, "transport": { "$ref": "#/$defs/sha256" }, - "imported_generation": { "$ref": "#/$defs/nonSecretIdentity" } + "generation_uuid": false } }, "package": { "type": "object", "additionalProperties": false, "required": ["contract", "format", "class", "integrity", "compatibility", "policy"], "properties": { "contract": { "const": "graphforge-portable-verify/2" }, "format": { "const": "portable-project-v2-bundle" }, "class": { "const": "complete" }, "integrity": { "const": "verified" }, "compatibility": { "const": "supported" }, "policy": { "const": "complete-current-generation" } } }, "authority": { "type": "object", "required": ["source_fingerprint", "imported_fingerprint"], "properties": { "source_fingerprint": { "$ref": "#/$defs/sha256" }, "imported_fingerprint": { "$ref": "#/$defs/sha256" } }, "additionalProperties": false }, "equivalence": { "type": "object", "additionalProperties": false, "required": ["source_project_fingerprint", "imported_project_fingerprint"], "properties": { "source_project_fingerprint": { "$ref": "#/$defs/sha256" }, "imported_project_fingerprint": { "$ref": "#/$defs/sha256" } } }, + "storage_attribution": { "$ref": "#/$defs/storageAttribution" }, "phases": { "type": "array", "minItems": 17, "maxItems": 32, "items": { "$ref": "#/$defs/phase" } }, - "envelope": { "type": "object", "required": ["peak_rss_bytes", "peak_disk_bytes", "wall_time_s"], "properties": { "peak_rss_bytes": { "type": "integer", "maximum": 137438953472 }, "peak_disk_bytes": { "type": "integer", "maximum": 1099511627776 }, "wall_time_s": { "type": "number", "maximum": 14400 } }, "additionalProperties": false }, + "envelope": { "type": "object", "required": ["peak_rss_bytes", "peak_disk_bytes", "peak_disk_source", "wall_time_s"], "properties": { "peak_rss_bytes": { "type": "integer", "maximum": 137438953472 }, "peak_disk_bytes": { "type": "integer", "maximum": 1099511627776 }, "peak_disk_source": { "const": "storage_owned_active_identity_union" }, "wall_time_s": { "type": "number", "maximum": 14400 } }, "additionalProperties": false }, "result": { "enum": ["pass", "fail"] }, "first_failure": { "type": ["object", "null"] } }, "$defs": { "sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, - "nonSecretIdentity": { "type": "string", "pattern": "^[0-9a-f-]{32,64}$" }, + "nonNegative": { "type": "integer", "minimum": 0 }, + "artifactTotals": { + "type": "object", "additionalProperties": false, + "required": ["logical_references", "logical_bytes", "physical_objects", "physical_logical_bytes", "allocated_bytes"], + "properties": { "logical_references": { "$ref": "#/$defs/nonNegative" }, "logical_bytes": { "$ref": "#/$defs/nonNegative" }, "physical_objects": { "$ref": "#/$defs/nonNegative" }, "physical_logical_bytes": { "$ref": "#/$defs/nonNegative" }, "allocated_bytes": { "$ref": "#/$defs/nonNegative" } } + }, + "artifactCategories": { + "type": "object", "additionalProperties": false, + "required": ["topology_nodes", "topology_edges", "properties", "uuid_and_surrogates", "adjacency", "catalog_and_manifests", "construction_staging", "portable_package", "clean_imported_project", "other"], + "properties": { + "topology_nodes": { "$ref": "#/$defs/artifactTotals" }, "topology_edges": { "$ref": "#/$defs/artifactTotals" }, "properties": { "$ref": "#/$defs/artifactTotals" }, "uuid_and_surrogates": { "$ref": "#/$defs/artifactTotals" }, "adjacency": { "$ref": "#/$defs/artifactTotals" }, "catalog_and_manifests": { "$ref": "#/$defs/artifactTotals" }, "construction_staging": { "$ref": "#/$defs/artifactTotals" }, "portable_package": { "$ref": "#/$defs/artifactTotals" }, "clean_imported_project": { "$ref": "#/$defs/artifactTotals" }, "other": { "$ref": "#/$defs/artifactTotals" } + } + }, + "snapshot": { + "type": "object", "additionalProperties": false, + "required": ["generation_manifest_sha256", "categories", "logical_references", "logical_bytes", "physical_objects", "physical_logical_bytes", "allocated_bytes"], + "properties": { "generation_manifest_sha256": { "type": "array", "minItems": 32, "maxItems": 32, "items": { "type": "integer", "minimum": 0, "maximum": 255 } }, "categories": { "$ref": "#/$defs/artifactCategories" }, "logical_references": { "$ref": "#/$defs/nonNegative" }, "logical_bytes": { "$ref": "#/$defs/nonNegative" }, "physical_objects": { "$ref": "#/$defs/nonNegative" }, "physical_logical_bytes": { "$ref": "#/$defs/nonNegative" }, "allocated_bytes": { "$ref": "#/$defs/nonNegative" } } + }, + "portableAllocation": { + "type": "object", "additionalProperties": false, + "required": ["category", "logical_bytes", "allocated_bytes", "logical_references", "physical_objects", "source"], + "properties": { "category": { "const": "portable_package" }, "logical_bytes": { "$ref": "#/$defs/nonNegative" }, "allocated_bytes": { "$ref": "#/$defs/nonNegative" }, "logical_references": { "$ref": "#/$defs/nonNegative" }, "physical_objects": { "$ref": "#/$defs/nonNegative" }, "source": { "const": "portable_writer_receipt" } } + }, + "phaseTotals": { + "type": "object", "additionalProperties": false, + "required": ["read_bytes", "write_bytes", "read_calls", "write_calls", "object_count", "block_count", "fsync_calls"], + "properties": { "read_bytes": { "$ref": "#/$defs/nonNegative" }, "write_bytes": { "$ref": "#/$defs/nonNegative" }, "read_calls": { "$ref": "#/$defs/nonNegative" }, "write_calls": { "$ref": "#/$defs/nonNegative" }, "object_count": { "$ref": "#/$defs/nonNegative" }, "block_count": { "$ref": "#/$defs/nonNegative" }, "fsync_calls": { "$ref": "#/$defs/nonNegative" } } + }, + "phaseMap": { + "type": "object", "additionalProperties": false, + "required": ["append_merge", "seal_authentication", "shape_consume_reauthentication", "encode_write_postwrite_authentication", "publication_preauthentication", "cas_install_read_write", "hydration_verification", "fsync_synchronization", "recovery_reauthentication"], + "properties": { "append_merge": { "$ref": "#/$defs/phaseTotals" }, "seal_authentication": { "$ref": "#/$defs/phaseTotals" }, "shape_consume_reauthentication": { "$ref": "#/$defs/phaseTotals" }, "encode_write_postwrite_authentication": { "$ref": "#/$defs/phaseTotals" }, "publication_preauthentication": { "$ref": "#/$defs/phaseTotals" }, "cas_install_read_write": { "$ref": "#/$defs/phaseTotals" }, "hydration_verification": { "$ref": "#/$defs/phaseTotals" }, "fsync_synchronization": { "$ref": "#/$defs/phaseTotals" }, "recovery_reauthentication": { "$ref": "#/$defs/phaseTotals" } } + }, + "phaseAttribution": { + "type": "object", "additionalProperties": false, "required": ["phases", "totals"], + "properties": { "phases": { "$ref": "#/$defs/phaseMap" }, "totals": { "$ref": "#/$defs/phaseTotals" } } + }, + "construction": { + "type": "object", "additionalProperties": false, + "required": ["seal_application_read_bytes", "shape_application_read_bytes", "encode_application_read_bytes", "encode_application_read_operations", "encode_application_write_bytes", "encode_application_write_operations", "encode_fsync_operations", "publication_application_read_bytes", "publication_application_read_operations", "cas_application_read_bytes", "cas_application_read_operations", "cas_application_write_bytes", "cas_application_write_operations", "cas_fsync_operations", "hydration_application_read_bytes", "hydration_application_read_operations", "hydration_application_write_bytes", "hydration_application_write_operations", "hydration_fsync_operations", "recovery_application_read_bytes", "recovery_application_read_operations", "canonical_output_bytes", "staged_and_retained_disk_bytes", "storage_current", "storage_transient_peak_allocated_bytes", "storage_transient_peak_total_allocated_bytes", "input_rows", "input_batches", "parquet_shards", "write_bytes", "write_operations", "fsync_operations", "authentication_read_bytes", "authentication_read_operations", "parent_catalog_read_bytes", "parent_catalog_read_operations", "retained_probe_read_bytes", "retained_probe_block_loads", "shaped_output_authentication_bytes", "shaped_output_authentication_operations", "replay_validation_read_bytes", "replay_validation_read_operations", "shape_input_validation_read_bytes", "shape_input_validation_read_operations", "run_records", "peak_batch_rows", "peak_batch_bytes", "peak_run_records", "prior_topology_rows_decoded", "current_transitions", "replayed_chunks", "merge_read_records", "merge_read_operations", "merge_written_records", "merge_write_operations", "merge_groups", "peak_merge_inputs", "merge_read_bytes", "merge_written_bytes", "merge_read_blocks", "merge_write_blocks", "merge_passes", "peak_merge_temporary_bytes", "current_merge_temporary_allocated_bytes", "peak_accounted_live_bytes", "peak_merge_name_slots", "peak_resolved_endpoint_name_slots", "peak_catalog_entries", "peak_catalog_identifier_bytes", "peak_catalog_decoded_batch_bytes", "merge_fsync_operations", "parquet_read_bytes", "parquet_read_operations", "parquet_write_bytes", "parquet_write_operations"], + "properties": { + "seal_application_read_bytes": { "$ref": "#/$defs/nonNegative" }, "shape_application_read_bytes": { "$ref": "#/$defs/nonNegative" }, "encode_application_read_bytes": { "$ref": "#/$defs/nonNegative" }, "encode_application_read_operations": { "$ref": "#/$defs/nonNegative" }, "encode_application_write_bytes": { "$ref": "#/$defs/nonNegative" }, "encode_application_write_operations": { "$ref": "#/$defs/nonNegative" }, "encode_fsync_operations": { "$ref": "#/$defs/nonNegative" }, "publication_application_read_bytes": { "$ref": "#/$defs/nonNegative" }, "publication_application_read_operations": { "$ref": "#/$defs/nonNegative" }, "cas_application_read_bytes": { "$ref": "#/$defs/nonNegative" }, "cas_application_read_operations": { "$ref": "#/$defs/nonNegative" }, "cas_application_write_bytes": { "$ref": "#/$defs/nonNegative" }, "cas_application_write_operations": { "$ref": "#/$defs/nonNegative" }, "cas_fsync_operations": { "$ref": "#/$defs/nonNegative" }, "hydration_application_read_bytes": { "$ref": "#/$defs/nonNegative" }, "hydration_application_read_operations": { "$ref": "#/$defs/nonNegative" }, "hydration_application_write_bytes": { "$ref": "#/$defs/nonNegative" }, "hydration_application_write_operations": { "$ref": "#/$defs/nonNegative" }, "hydration_fsync_operations": { "$ref": "#/$defs/nonNegative" }, "recovery_application_read_bytes": { "$ref": "#/$defs/nonNegative" }, "recovery_application_read_operations": { "$ref": "#/$defs/nonNegative" }, "canonical_output_bytes": { "$ref": "#/$defs/nonNegative" }, "staged_and_retained_disk_bytes": { "$ref": "#/$defs/nonNegative" }, "storage_current": { "$ref": "#/$defs/artifactCategories" }, "storage_transient_peak_allocated_bytes": { "type": "object", "additionalProperties": false, "required": ["topology_nodes", "topology_edges", "properties", "uuid_and_surrogates", "adjacency", "catalog_and_manifests", "construction_staging", "portable_package", "clean_imported_project", "other"], "properties": { "topology_nodes": { "$ref": "#/$defs/nonNegative" }, "topology_edges": { "$ref": "#/$defs/nonNegative" }, "properties": { "$ref": "#/$defs/nonNegative" }, "uuid_and_surrogates": { "$ref": "#/$defs/nonNegative" }, "adjacency": { "$ref": "#/$defs/nonNegative" }, "catalog_and_manifests": { "$ref": "#/$defs/nonNegative" }, "construction_staging": { "$ref": "#/$defs/nonNegative" }, "portable_package": { "$ref": "#/$defs/nonNegative" }, "clean_imported_project": { "$ref": "#/$defs/nonNegative" }, "other": { "$ref": "#/$defs/nonNegative" } } }, "storage_transient_peak_total_allocated_bytes": { "$ref": "#/$defs/nonNegative" }, + "input_rows": { "$ref": "#/$defs/nonNegative" }, "input_batches": { "$ref": "#/$defs/nonNegative" }, "parquet_shards": { "$ref": "#/$defs/nonNegative" }, "write_bytes": { "$ref": "#/$defs/nonNegative" }, "write_operations": { "$ref": "#/$defs/nonNegative" }, "fsync_operations": { "$ref": "#/$defs/nonNegative" }, "authentication_read_bytes": { "$ref": "#/$defs/nonNegative" }, "authentication_read_operations": { "$ref": "#/$defs/nonNegative" }, "parent_catalog_read_bytes": { "$ref": "#/$defs/nonNegative" }, "parent_catalog_read_operations": { "$ref": "#/$defs/nonNegative" }, "retained_probe_read_bytes": { "$ref": "#/$defs/nonNegative" }, "retained_probe_block_loads": { "$ref": "#/$defs/nonNegative" }, "shaped_output_authentication_bytes": { "$ref": "#/$defs/nonNegative" }, "shaped_output_authentication_operations": { "$ref": "#/$defs/nonNegative" }, "replay_validation_read_bytes": { "$ref": "#/$defs/nonNegative" }, "replay_validation_read_operations": { "$ref": "#/$defs/nonNegative" }, "shape_input_validation_read_bytes": { "$ref": "#/$defs/nonNegative" }, "shape_input_validation_read_operations": { "$ref": "#/$defs/nonNegative" }, "run_records": { "$ref": "#/$defs/nonNegative" }, "peak_batch_rows": { "$ref": "#/$defs/nonNegative" }, "peak_batch_bytes": { "$ref": "#/$defs/nonNegative" }, "peak_run_records": { "$ref": "#/$defs/nonNegative" }, "prior_topology_rows_decoded": { "$ref": "#/$defs/nonNegative" }, "current_transitions": { "$ref": "#/$defs/nonNegative" }, "replayed_chunks": { "$ref": "#/$defs/nonNegative" }, "merge_read_records": { "$ref": "#/$defs/nonNegative" }, "merge_read_operations": { "$ref": "#/$defs/nonNegative" }, "merge_written_records": { "$ref": "#/$defs/nonNegative" }, "merge_write_operations": { "$ref": "#/$defs/nonNegative" }, "merge_groups": { "$ref": "#/$defs/nonNegative" }, "peak_merge_inputs": { "$ref": "#/$defs/nonNegative" }, "merge_read_bytes": { "$ref": "#/$defs/nonNegative" }, "merge_written_bytes": { "$ref": "#/$defs/nonNegative" }, "merge_read_blocks": { "$ref": "#/$defs/nonNegative" }, "merge_write_blocks": { "$ref": "#/$defs/nonNegative" }, "merge_passes": { "$ref": "#/$defs/nonNegative" }, "peak_merge_temporary_bytes": { "$ref": "#/$defs/nonNegative" }, "current_merge_temporary_allocated_bytes": { "$ref": "#/$defs/nonNegative" }, "peak_accounted_live_bytes": { "$ref": "#/$defs/nonNegative" }, "peak_merge_name_slots": { "$ref": "#/$defs/nonNegative" }, "peak_resolved_endpoint_name_slots": { "$ref": "#/$defs/nonNegative" }, "peak_catalog_entries": { "$ref": "#/$defs/nonNegative" }, "peak_catalog_identifier_bytes": { "$ref": "#/$defs/nonNegative" }, "peak_catalog_decoded_batch_bytes": { "$ref": "#/$defs/nonNegative" }, "merge_fsync_operations": { "$ref": "#/$defs/nonNegative" }, "parquet_read_bytes": { "$ref": "#/$defs/nonNegative" }, "parquet_read_operations": { "$ref": "#/$defs/nonNegative" }, "parquet_write_bytes": { "$ref": "#/$defs/nonNegative" }, "parquet_write_operations": { "$ref": "#/$defs/nonNegative" } + } + }, + "storageAttribution": { + "type": "object", "additionalProperties": false, + "required": ["source", "source_project_current_allocated_bytes", "portable_package", "clean_import", "construction", "application_io_phases", "workspace_current_allocated_bytes"], + "properties": { "source": { "$ref": "#/$defs/snapshot" }, "source_project_current_allocated_bytes": { "$ref": "#/$defs/nonNegative" }, "portable_package": { "$ref": "#/$defs/portableAllocation" }, "clean_import": { "$ref": "#/$defs/snapshot" }, "construction": { "$ref": "#/$defs/construction" }, "application_io_phases": { "$ref": "#/$defs/phaseAttribution" }, "workspace_current_allocated_bytes": { "$ref": "#/$defs/nonNegative" } } + }, "phase": { "type": "object", "additionalProperties": false, "required": ["id", "status", "elapsed_ms", "rss_peak_bytes", "disk_peak_bytes"], "properties": { "id": { "type": "string", "pattern": "^[a-z0-9_-]+$" }, "status": { "enum": ["pass", "fail", "cancelled"] }, "elapsed_ms": { "type": "integer", "minimum": 0 }, "rss_peak_bytes": { "type": "integer", "minimum": 0 }, "disk_peak_bytes": { "type": "integer", "minimum": 0 }, "fingerprint": { "anyOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] }, "failure_code": { "type": ["string", "null"], "maxLength": 96 } } } } } diff --git a/docs/development/evidence/g500-ladder-qualification.schema.json b/docs/development/evidence/g500-ladder-qualification.schema.json new file mode 100644 index 000000000..80f88d31d --- /dev/null +++ b/docs/development/evidence/g500-ladder-qualification.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://graphforge.dev/schemas/g500-ladder-qualification-3.json", + "type": "object", "additionalProperties": false, + "required": ["schema", "rungs", "projection"], + "properties": { + "schema": {"const": "graphforge-g500-ladder-qualification/3"}, + "rungs": {"type": "array", "minItems": 2, "maxItems": 4, "items": {"$ref": "#/$defs/rung"}}, + "projection": {"$ref": "#/$defs/projection"} + }, + "$defs": { + "nonNegative": {"type": "integer", "minimum": 0}, "positive": {"type": "integer", "minimum": 1}, + "ratio": {"type": "object", "additionalProperties": false, "required": ["numerator_bytes", "denominator_count"], "properties": {"numerator_bytes": {"$ref": "#/$defs/nonNegative"}, "denominator_count": {"$ref": "#/$defs/positive"}}}, + "artifact": { + "type": "object", "additionalProperties": false, + "required": ["category", "logical_bytes", "allocated_bytes", "current_retained_bytes", "transient_peak_allocated_bytes", "logical_references", "physical_objects", "source"], + "properties": { + "category": {"enum": ["canonical_node_topology", "canonical_edge_topology", "properties", "uuid_surrogate_indexes", "adjacency_csr", "catalog_manifests", "construction_staging_spill", "portable_package", "clean_imported_project"]}, + "logical_bytes": {"$ref": "#/$defs/nonNegative"}, "allocated_bytes": {"$ref": "#/$defs/nonNegative"}, + "current_retained_bytes": {"$ref": "#/$defs/nonNegative"}, "transient_peak_allocated_bytes": {"$ref": "#/$defs/nonNegative"}, + "logical_references": {"$ref": "#/$defs/nonNegative"}, "physical_objects": {"$ref": "#/$defs/nonNegative"}, + "source": {"enum": ["storage_owned_snapshot", "construction_receipts", "exact_descriptor", "clean_import_snapshot"]} + } + }, + "phase": { + "type": "object", "additionalProperties": false, + "required": ["phase", "applicable", "read_bytes", "write_bytes", "read_calls", "write_calls", "object_count", "block_count", "fsync_calls"], + "properties": { + "phase": {"enum": ["append_merge", "seal_authentication", "shape_consume_reauthentication", "encode_write_postwrite_authentication", "publication_preauthentication", "cas_install_read_write", "hydration_verification", "fsync_synchronization", "recovery_reauthentication"]}, + "applicable": {"type": "boolean"}, + "read_bytes": {"$ref": "#/$defs/nonNegative"}, "write_bytes": {"$ref": "#/$defs/nonNegative"}, "read_calls": {"$ref": "#/$defs/nonNegative"}, "write_calls": {"$ref": "#/$defs/nonNegative"}, + "object_count": {"$ref": "#/$defs/nonNegative"}, "block_count": {"$ref": "#/$defs/nonNegative"}, "fsync_calls": {"$ref": "#/$defs/nonNegative"} + } + }, + "totals": { + "type": "object", "additionalProperties": false, + "required": ["logical_bytes", "allocated_bytes", "current_retained_bytes", "transient_peak_allocated_bytes", "phase_read_bytes", "phase_write_bytes", "phase_read_calls", "phase_write_calls", "phase_object_count", "phase_block_count", "phase_fsync_calls"], + "properties": { + "logical_bytes": {"$ref": "#/$defs/nonNegative"}, "allocated_bytes": {"$ref": "#/$defs/nonNegative"}, "current_retained_bytes": {"$ref": "#/$defs/nonNegative"}, "transient_peak_allocated_bytes": {"$ref": "#/$defs/nonNegative"}, + "phase_read_bytes": {"$ref": "#/$defs/nonNegative"}, "phase_write_bytes": {"$ref": "#/$defs/nonNegative"}, "phase_read_calls": {"$ref": "#/$defs/nonNegative"}, "phase_write_calls": {"$ref": "#/$defs/nonNegative"}, + "phase_object_count": {"$ref": "#/$defs/nonNegative"}, "phase_block_count": {"$ref": "#/$defs/nonNegative"}, "phase_fsync_calls": {"$ref": "#/$defs/nonNegative"} + } + }, + "rung": { + "type": "object", "additionalProperties": false, + "required": ["id", "scale", "live_nodes", "live_edges", "source_project_current_allocated_bytes", "workspace_current_allocated_bytes", "artifacts", "phases", "totals", "ratios"], + "properties": { + "id": {"enum": ["S20", "S22", "S24", "S26"]}, "scale": {"enum": [20, 22, 24, 26]}, "live_nodes": {"$ref": "#/$defs/positive"}, "live_edges": {"$ref": "#/$defs/positive"}, + "source_project_current_allocated_bytes": {"$ref": "#/$defs/nonNegative"}, "workspace_current_allocated_bytes": {"$ref": "#/$defs/nonNegative"}, + "artifacts": {"type": "array", "minItems": 9, "maxItems": 9, "items": {"$ref": "#/$defs/artifact"}}, "phases": {"type": "array", "minItems": 9, "maxItems": 9, "items": {"$ref": "#/$defs/phase"}}, + "totals": {"$ref": "#/$defs/totals"}, + "ratios": {"type": "object", "additionalProperties": false, "required": ["canonical_node_bytes_per_live_node", "canonical_edge_bytes_per_live_edge", "authoritative_project_bytes_per_live_edge", "full_lifecycle_peak_bytes_per_live_edge"], "properties": { + "canonical_node_bytes_per_live_node": {"$ref": "#/$defs/ratio"}, "canonical_edge_bytes_per_live_edge": {"$ref": "#/$defs/ratio"}, "authoritative_project_bytes_per_live_edge": {"$ref": "#/$defs/ratio"}, "full_lifecycle_peak_bytes_per_live_edge": {"$ref": "#/$defs/ratio"} + }} + } + }, + "projection": { + "type": "object", "additionalProperties": false, + "required": ["target", "source_rungs", "rate", "projected_canonical_node_bytes", "projected_canonical_edge_bytes", "projected_lifecycle_peak_bytes", "volume_bytes", "reserved_headroom_bytes", "headroom_bytes", "decision"], + "properties": { + "target": {"const": "S26"}, "source_rungs": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"enum": ["S20", "S22", "S24"]}}, "rate": {"$ref": "#/$defs/ratio"}, + "projected_canonical_node_bytes": {"$ref": "#/$defs/nonNegative"}, "projected_canonical_edge_bytes": {"$ref": "#/$defs/nonNegative"}, "projected_lifecycle_peak_bytes": {"$ref": "#/$defs/nonNegative"}, "volume_bytes": {"$ref": "#/$defs/positive"}, + "reserved_headroom_bytes": {"$ref": "#/$defs/nonNegative"}, "headroom_bytes": {"$ref": "#/$defs/nonNegative"}, "decision": {"enum": ["admit", "refuse"]} + } + } + } +} diff --git a/docs/development/perf-g500-ladder.md b/docs/development/perf-g500-ladder.md index bd191997c..48f64c8ed 100644 --- a/docs/development/perf-g500-ladder.md +++ b/docs/development/perf-g500-ladder.md @@ -119,7 +119,9 @@ therefore survive process replacement and are reused on re-entry. > windows into immutable Parquet shards and retains only bounded merge/probe > state; accumulated topology remains disk-owned. While ingest runs, the atomic journal is > refreshed every two seconds with the current subphase, edge-chunk index, -> anonymous/file RSS, disk usage, and aggregate topology rewrite counters. An +> anonymous/file RSS, and aggregate topology rewrite counters. It deliberately +> does not recursively walk the active project. Disk attribution comes from +> storage-owned counters and exact descriptors at completed phase boundaries. An > `oom` with `first_failing_phase: "ingest"` therefore remains an upstream > construction failure, not a generator-memory regression. Each completed > ingest phase records elapsed time, RSS, disk bytes, shard count, input rows, @@ -207,6 +209,104 @@ Wall-clock and RSS numbers are hardware-specific observations, never CI millisecond gates. For #745, `sut` must name the cloud SKU; laptop SUTs are rejected as certification evidence. +### Disk attribution and S26 admission + +The versioned `graphforge-g500-ladder-qualification/3` companion document is +validated by `scripts/ci/validate-g500-ladder-qualification.py`. Every observed +rung has exactly one row for canonical node topology, canonical edge topology, +properties, UUID/surrogate indexes, adjacency/CSR, catalogs/manifests, +construction staging/spill, portable package, and clean import. Native file +identity deduplicates content-addressed/shared objects. Logical bytes, +filesystem allocation, current retained allocation, and full-lifecycle +transient peak remain separate quantities. + +Artifact rows are local ownership views and may refer to the same physical CAS +object. Their allocated/current columns therefore are not summed to obtain the +workspace footprint. `totals.current_retained_bytes` is the independently +reconciled native-identity union across all simultaneously retained owners. +The authoritative-project ratio uses a separate source-project native-identity +union captured at the stable source boundary. It includes the project controls, +every retained generation, and every CAS object not yet removed by an exact GC +receipt. It is therefore bounded below by the selected-generation snapshot and +above by the independent workspace union, but it does not include construction +staging, the portable package, drills, or the clean imported project. + +The lifecycle peak is not reconstructed by adding category peaks or directory +sizes. Storage owns a reference-counted union keyed by authenticated native +`(volume, file-id)` identities. Append, shaping, encoding, CAS publication, +portable export, private import materialization, clean publication, corruption +drills, and interrupted-import cleanup install or remove exact owners. The +high-water mark advances at each transition, so aliases count once and files +that did not coexist are never added together. Certification emits +`storage_owned_active_identity_union` provenance only from this tracker. + +The project owner includes `FORMAT`, `CURRENT`, and every authenticated +generation still installed in the bounded generation namespace, including +checkpoint branches and generations not yet reclaimed; publication does not +discard an old generation from accounting merely because `CURRENT` advanced. +It also includes the CAS lifecycle control and every sealed CAS object, including +objects not referenced by the current generation. Only an exact explicit GC +receipt may remove those identities. The bounded CAS inventory runs at retained +phase boundaries, never during active ingest. Portable writers record native allocation as files +are written, synchronized, published, or removed; they never rediscover a +large export with a recursive post-write directory pass, and measurement +failure is a typed operation failure rather than a zero observation. +Portable import success additionally carries an identity-safe cleanup receipt; +the materialization owner is removed only after authenticated deletion and +parent-directory synchronization complete. Cleanup failure is a typed import +failure and its still-owned identities remain in lifecycle evidence. + +The same document carries a closed nine-phase inventory: append/merge, seal +authentication, shape consumption/reauthentication, encode plus post-write +authentication, publication preauthentication, CAS install, hydration +verification, synchronization, and recovery reauthentication. Raw bytes, +calls, blocks, objects, and fsyncs reconcile exactly before ratios are derived. +Fixed-run merge bytes and calls come from the same instrumented readers and +writers: `merge_read_operations` and `merge_write_operations` count actual +non-empty storage submissions, while block counters remain a separate transfer +granularity metric. Shape-phase totals add the disjoint fixed-run and Parquet +byte/call counters exactly once. +Each phase declares whether it was applicable. Every ordinary lifecycle phase +must contain source-owned activity; a zero row is rejected. Recovery may be +non-applicable only for an uninterrupted run, while the deterministic durable +crash matrix separately proves nonzero recovery bytes and calls whenever an +interrupted intent is accepted. +The deterministic full-lifecycle 1x/2x/4x ladder executes source construction, +CSR, reopen/query, export/verify, clean import/reopen/query, corruption, +cancellation, resource-limit, and interrupted-finalization drills. It validates +the qualification phase inventory and bounds bytes, calls, objects, blocks, and +fsyncs per phase rather than relying only on aggregate I/O. +Node canonical cost uses reopened live nodes; edge canonical, authoritative +project, and lifecycle peak costs use reopened live edges. Ratios preserve raw +integer numerators and denominators; rounded decimals are not evidence. + +Provider and qualification artifacts never expose generation UUIDs. Generation +agreement and source/import distinctness are checked while the generations are +lifetime-pinned, then emitted only as required-true authenticated proof fields. +Before either artifact is written, a recursive sanitizer rejects raw UUID +strings, absolute host paths, credentials, secrets, tokens, and provider +machine, volume, or resource identifiers. Storage snapshots likewise omit the +generation UUID and native file-identity map. + +At least two ordered adjacent rungs are required. The S26 rate must be no lower +than both the newest observed peak ratio and every positive adjacent-rung slope. +The validator independently recomputes separate projected canonical-node and +canonical-edge allocation plus the lifecycle peak, +volume headroom, and the admit/refuse decision. A single successful rung is not +a projection, and insufficient reserved headroom always refuses SCALE-26. + +Build and validate a companion document from two adjacent real certification +documents with: + +```bash +make g500-ladder-qualification \ + LOW_CERT=build/s20-certification.json \ + HIGH_CERT=build/s22-certification.json \ + EVIDENCE=build/g500-ladder-qualification.json \ + VOLUME_BYTES=536870912000 \ + RESERVED_HEADROOM_BYTES=53687091200 +``` + ## CI placement Per the [Scale Evaluation](../reference/scale-evaluation.md) contract, large diff --git a/scripts/ci/build-g500-ladder-qualification.py b/scripts/ci/build-g500-ladder-qualification.py new file mode 100644 index 000000000..c9ae9b6c9 --- /dev/null +++ b/scripts/ci/build-g500-ladder-qualification.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Build sanitized #951 qualification evidence from adjacent certifications.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re + +FORBIDDEN_KEY = re.compile( + r"(?:secret|credential|token|password|host_path|absolute_path|machine[_-]?id|volume[_-]?id|provider_resource_id)", + re.I, +) +ABSOLUTE_PATH = re.compile(r"(?:^|[\s=:])(?:/|[A-Za-z]:[\\/])") +RAW_UUID = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + re.I, +) + + +def reject_unsanitized(value, trail="$") -> None: + if isinstance(value, dict): + for key, child in value.items(): + if FORBIDDEN_KEY.search(key): + raise ValueError(f"sensitive evidence key at {trail}.{key}") + reject_unsanitized(child, f"{trail}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + reject_unsanitized(child, f"{trail}[{index}]") + elif isinstance(value, str): + if ABSOLUTE_PATH.search(value): + raise ValueError(f"absolute host path at {trail}") + if RAW_UUID.fullmatch(value): + raise ValueError(f"raw UUID at {trail}") + + +CATEGORIES = ( + ("canonical_node_topology", "topology_nodes", "storage_owned_snapshot"), + ("canonical_edge_topology", "topology_edges", "storage_owned_snapshot"), + ("properties", "properties", "storage_owned_snapshot"), + ("uuid_surrogate_indexes", "uuid_and_surrogates", "storage_owned_snapshot"), + ("adjacency_csr", "adjacency", "storage_owned_snapshot"), + ("catalog_manifests", "catalog_and_manifests", "storage_owned_snapshot"), +) + + +def artifact(category: str, totals: dict, source: str, peak: int | None = None) -> dict: + allocated = totals.get("allocated_bytes", 0) + return { + "category": category, + "logical_bytes": totals.get("logical_bytes", totals.get("physical_logical_bytes", 0)), + "allocated_bytes": allocated, + "current_retained_bytes": allocated, + "transient_peak_allocated_bytes": allocated if peak is None else peak, + "logical_references": totals.get("logical_references", 0), + "physical_objects": totals.get("physical_objects", 0), + "source": source, + } + + +def rung(cert: dict) -> dict: + if cert.get("envelope", {}).get("peak_disk_source") != "storage_owned_active_identity_union": + raise ValueError("certification peak disk is not a storage-owned active identity union") + storage = cert["storage_attribution"] + source = storage["source"] + rows = [artifact(name, source["categories"][key], owner) for name, key, owner in CATEGORIES] + construction = storage["construction"] + staging = construction.get("storage_current", {}).get("construction_staging", {}) + rows.append( + artifact( + "construction_staging_spill", + staging, + "construction_receipts", + construction.get("storage_transient_peak_total_allocated_bytes", 0), + ) + ) + rows.append(artifact("portable_package", storage["portable_package"], "exact_descriptor")) + rows.append( + artifact("clean_imported_project", storage["clean_import"], "clean_import_snapshot") + ) + phase_map = storage["application_io_phases"]["phases"] + phases = [] + for name, values in phase_map.items(): + applicable = any( + values[field] != 0 + for field in ( + "read_bytes", + "write_bytes", + "read_calls", + "write_calls", + "object_count", + "block_count", + "fsync_calls", + ) + ) + phases.append({"phase": name, "applicable": applicable, **values}) + totals = { + "logical_bytes": sum(row["logical_bytes"] for row in rows), + "allocated_bytes": sum(row["allocated_bytes"] for row in rows), + # This is the native-identity union across simultaneously retained + # owners. Category rows are local ownership views and can alias the + # same CAS object, so summing them would double count. + "current_retained_bytes": storage["workspace_current_allocated_bytes"], + "transient_peak_allocated_bytes": cert["envelope"]["peak_disk_bytes"], + } + for field in ( + "read_bytes", + "write_bytes", + "read_calls", + "write_calls", + "object_count", + "block_count", + "fsync_calls", + ): + totals[f"phase_{field}"] = sum(phase[field] for phase in phases) + nodes, edges = cert["counts"]["source_nodes"], cert["counts"]["source_edges"] + by_name = {row["category"]: row for row in rows} + return { + "id": f"S{cert['run']['scale']}", + "scale": cert["run"]["scale"], + "live_nodes": nodes, + "live_edges": edges, + "source_project_current_allocated_bytes": storage["source_project_current_allocated_bytes"], + "workspace_current_allocated_bytes": storage["workspace_current_allocated_bytes"], + "artifacts": rows, + "phases": phases, + "totals": totals, + "ratios": { + "canonical_node_bytes_per_live_node": { + "numerator_bytes": by_name["canonical_node_topology"]["logical_bytes"], + "denominator_count": nodes, + }, + "canonical_edge_bytes_per_live_edge": { + "numerator_bytes": by_name["canonical_edge_topology"]["logical_bytes"], + "denominator_count": edges, + }, + "authoritative_project_bytes_per_live_edge": { + "numerator_bytes": storage["source_project_current_allocated_bytes"], + "denominator_count": edges, + }, + "full_lifecycle_peak_bytes_per_live_edge": { + "numerator_bytes": totals["transient_peak_allocated_bytes"], + "denominator_count": edges, + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("low", type=Path) + parser.add_argument("high", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--volume-bytes", type=int, required=True) + parser.add_argument("--reserved-headroom-bytes", type=int, required=True) + args = parser.parse_args() + certifications = [json.loads(path.read_text()) for path in (args.low, args.high)] + for certification in certifications: + reject_unsanitized(certification) + rungs = [rung(certification) for certification in certifications] + low, high = rungs + delta_bytes = ( + high["totals"]["transient_peak_allocated_bytes"] + - low["totals"]["transient_peak_allocated_bytes"] + ) + delta_edges = high["live_edges"] - low["live_edges"] + ratio_num, ratio_den = high["totals"]["transient_peak_allocated_bytes"], high["live_edges"] + if delta_bytes > 0 and delta_bytes * ratio_den > ratio_num * delta_edges: + ratio_num, ratio_den = delta_bytes, delta_edges + target_edges = 1 << 30 + peak = (ratio_num * target_edges + ratio_den - 1) // ratio_den + target_nodes = 1 << 26 + by_category = {row["category"]: row for row in high["artifacts"]} + canonical_nodes = ( + by_category["canonical_node_topology"]["current_retained_bytes"] * target_nodes + + high["live_nodes"] + - 1 + ) // high["live_nodes"] + canonical_edges = ( + by_category["canonical_edge_topology"]["current_retained_bytes"] * target_edges + + high["live_edges"] + - 1 + ) // high["live_edges"] + headroom = max(0, args.volume_bytes - peak) + decision = ( + "admit" + if peak <= args.volume_bytes and headroom >= args.reserved_headroom_bytes + else "refuse" + ) + value = { + "schema": "graphforge-g500-ladder-qualification/3", + "rungs": rungs, + "projection": { + "target": "S26", + "source_rungs": [low["id"], high["id"]], + "rate": {"numerator_bytes": ratio_num, "denominator_count": ratio_den}, + "projected_canonical_node_bytes": canonical_nodes, + "projected_canonical_edge_bytes": canonical_edges, + "projected_lifecycle_peak_bytes": peak, + "volume_bytes": args.volume_bytes, + "reserved_headroom_bytes": args.reserved_headroom_bytes, + "headroom_bytes": headroom, + "decision": decision, + }, + } + reject_unsanitized(value) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(value, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/test-non-cypher-surface-gate.py b/scripts/ci/test-non-cypher-surface-gate.py index 958edb164..9c3fb8062 100644 --- a/scripts/ci/test-non-cypher-surface-gate.py +++ b/scripts/ci/test-non-cypher-surface-gate.py @@ -29,7 +29,7 @@ def validate(self, manifest: dict) -> list[str]: def test_checked_in_inventory_is_complete(self) -> None: self.assertEqual(GATE.validate(), []) - self.assertEqual(len(GATE.public_methods()), 372) + self.assertEqual(len(GATE.public_methods()), 373) self.assertEqual(len(GATE.algorithm_registry()), 94) def test_new_or_removed_public_method_fails_frozen_digest(self) -> None: diff --git a/scripts/ci/test-validate-g500-certification.py b/scripts/ci/test-validate-g500-certification.py index cd0eb7198..5fcf53926 100644 --- a/scripts/ci/test-validate-g500-certification.py +++ b/scripts/ci/test-validate-g500-certification.py @@ -13,13 +13,100 @@ assert SPEC and SPEC.loader VALIDATOR = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(VALIDATOR) +BUILDER_SCRIPT = Path(__file__).with_name("build-g500-ladder-qualification.py") +BUILDER_SPEC = importlib.util.spec_from_file_location("g500_builder", BUILDER_SCRIPT) +assert BUILDER_SPEC and BUILDER_SPEC.loader +BUILDER = importlib.util.module_from_spec(BUILDER_SPEC) +BUILDER_SPEC.loader.exec_module(BUILDER) +QUALIFICATION_SCRIPT = Path(__file__).with_name("validate-g500-ladder-qualification.py") +QUALIFICATION_SPEC = importlib.util.spec_from_file_location( + "g500_qualification_validator", QUALIFICATION_SCRIPT +) +assert QUALIFICATION_SPEC and QUALIFICATION_SPEC.loader +QUALIFICATION = importlib.util.module_from_spec(QUALIFICATION_SPEC) +QUALIFICATION_SPEC.loader.exec_module(QUALIFICATION) SHA = "a" * 40 DIGEST_A = "sha256:" + "a" * 64 DIGEST_B = "sha256:" + "b" * 64 -def evidence(): +def artifact_totals(unit=1): + return { + "logical_references": unit, + "logical_bytes": unit, + "physical_objects": unit, + "physical_logical_bytes": unit, + "allocated_bytes": unit, + } + + +def storage_attribution(unit=1): + category_names = ( + "topology_nodes", + "topology_edges", + "properties", + "uuid_and_surrogates", + "adjacency", + "catalog_and_manifests", + "construction_staging", + "portable_package", + "clean_imported_project", + "other", + ) + categories = { + name: artifact_totals(unit if index < 6 else 0) for index, name in enumerate(category_names) + } + snapshot = { + "generation_manifest_sha256": [1] * 32, + "categories": categories, + "logical_references": 6 * unit, + "logical_bytes": 6 * unit, + "physical_objects": 6 * unit, + "physical_logical_bytes": 6 * unit, + "allocated_bytes": 6 * unit, + } + contract = json.loads(VALIDATOR.SCHEMA.read_text()) + construction = dict.fromkeys(contract["$defs"]["construction"]["required"], 0) + construction["storage_current"] = {name: artifact_totals(unit) for name in category_names} + construction["storage_transient_peak_allocated_bytes"] = dict.fromkeys(category_names, unit) + construction["storage_transient_peak_total_allocated_bytes"] = 10 * unit + phase_names = contract["$defs"]["phaseMap"]["required"] + phases = {} + for name in phase_names: + phases[name] = { + "read_bytes": unit, + "write_bytes": 0, + "read_calls": unit, + "write_calls": 0, + "object_count": 0, + "block_count": 0, + "fsync_calls": unit if name == "fsync_synchronization" else 0, + } + totals = { + field: sum(values[field] for values in phases.values()) + for field in next(iter(phases.values())) + } + return { + "source": snapshot, + "source_project_current_allocated_bytes": 7 * unit, + "portable_package": { + "category": "portable_package", + "logical_bytes": unit, + "allocated_bytes": unit, + "logical_references": unit, + "physical_objects": unit, + "source": "portable_writer_receipt", + }, + "clean_import": snapshot, + "construction": construction, + "application_io_phases": {"phases": phases, "totals": totals}, + "workspace_current_allocated_bytes": 14 * unit, + } + + +def evidence(scale=26, unit=1): + live = 1_000_000_000 if scale == 26 else (1 << scale) * 15 phases = [] for phase in VALIDATOR.REQUIRED_PHASES: fingerprint = DIGEST_A if "query_1hop" in phase else DIGEST_B @@ -39,7 +126,7 @@ def evidence(): "profile_sha256": "sha256:" + hashlib.sha256(VALIDATOR.PROFILE.read_bytes()).hexdigest(), "run": { "command": VALIDATOR.RUN_COMMAND, - "scale": 26, + "scale": scale, "edgefactor": 16, "seed": 1, "directionality": "undirected", @@ -54,25 +141,26 @@ def evidence(): "os": "Linux", "kernel": "6", "filesystem": "xfs", - "memory_bytes": 137_438_953_472, - "nvme_bytes": 1_099_511_627_776, + "memory_bytes": 4_294_967_296, + "nvme_bytes": 536_870_912_000, }, "tools": {"rustc": "1.90"}, "counts": { - "raw_attempts": 1_000_000_002, + "raw_attempts": live + 2, "self_loops_rejected": 1, "duplicates_rejected": 1, - "live_unique_edges": 1_000_000_000, - "source_nodes": 67_108_864, - "source_edges": 1_000_000_000, - "imported_nodes": 67_108_864, - "imported_edges": 1_000_000_000, + "live_unique_edges": live, + "source_nodes": 1 << scale, + "source_edges": live, + "imported_nodes": 1 << scale, + "imported_edges": live, }, "identities": { - "source_generation": "11111111-1111-1111-1111-111111111111", + "source_export_generation_authenticated": True, + "import_receipt_reopen_authenticated": True, + "source_import_generations_distinct": True, "package": DIGEST_A, "transport": DIGEST_B, - "imported_generation": "22222222-2222-2222-2222-222222222222", }, "package": { "contract": "graphforge-portable-verify/2", @@ -87,8 +175,14 @@ def evidence(): "source_project_fingerprint": DIGEST_A, "imported_project_fingerprint": DIGEST_A, }, + "storage_attribution": storage_attribution(unit), "phases": phases, - "envelope": {"peak_rss_bytes": 1, "peak_disk_bytes": 1, "wall_time_s": 1}, + "envelope": { + "peak_rss_bytes": 1, + "peak_disk_bytes": 100 * unit, + "peak_disk_source": "storage_owned_active_identity_union", + "wall_time_s": 1, + }, "result": "pass", "first_failure": None, } @@ -102,6 +196,78 @@ def test_accepts_complete_sanitized_evidence(): VALIDATOR.validate(evidence(), SHA) +def test_actual_certification_contract_builds_and_validates_adjacent_qualification( + tmp_path, monkeypatch +): + low = evidence(20, 1) + high = evidence(22, 4) + for document in (low, high): + VALIDATOR.validate(document, SHA) + low_path = tmp_path / "s20.json" + high_path = tmp_path / "s22.json" + output = tmp_path / "qualification.json" + low_path.write_text(json.dumps(low)) + high_path.write_text(json.dumps(high)) + monkeypatch.setattr( + "sys.argv", + [ + str(BUILDER_SCRIPT), + str(low_path), + str(high_path), + str(output), + "--volume-bytes", + str(500 * 1024**3), + "--reserved-headroom-bytes", + str(75 * 1024**3), + ], + ) + BUILDER.main() + qualification = json.loads(output.read_text()) + QUALIFICATION.validate(qualification) + assert qualification["projection"]["source_rungs"] == ["S20", "S22"] + for source, rung in zip((low, high), qualification["rungs"], strict=True): + selected = source["storage_attribution"]["source"]["allocated_bytes"] + project_union = rung["source_project_current_allocated_bytes"] + assert project_union > selected + assert rung["ratios"]["authoritative_project_bytes_per_live_edge"] == { + "numerator_bytes": project_union, + "denominator_count": rung["live_edges"], + } + + +@pytest.mark.parametrize( + ("section", "key", "value"), + [ + ("tools", "build", "018f6e45-7f12-7c00-8000-000000000001"), + ("tools", "build", "00000000-0000-0000-0000-000000000000"), + ("tools", "build", "/var/lib/graphforge/project"), + ("tools", "machine_id", "redacted"), + ("tools", "volume-id", "redacted"), + ("tools", "provider_resource_id", "redacted"), + ], +) +def test_recursive_sanitizer_rejects_raw_identity_path_and_sensitive_keys(section, key, value): + unsafe = evidence() + unsafe[section][key] = value + with pytest.raises(VALIDATOR.EvidenceError): + VALIDATOR.validate(unsafe, SHA) + + +@pytest.mark.parametrize( + "proof", + [ + "source_export_generation_authenticated", + "import_receipt_reopen_authenticated", + "source_import_generations_distinct", + ], +) +def test_generation_proofs_are_closed_and_required_true(proof): + unsafe = evidence() + unsafe["identities"][proof] = False + with pytest.raises(VALIDATOR.EvidenceError): + VALIDATOR.validate(unsafe, SHA) + + @pytest.mark.parametrize( "mutation", [ @@ -144,7 +310,7 @@ def test_rejects_incomplete_or_unsafe_evidence(mutation): if mutation == "run": value["run"]["seed"] = 2 if mutation == "identity": - value["identities"]["imported_generation"] = value["identities"]["source_generation"] + value["identities"]["source_import_generations_distinct"] = False if mutation == "authority": value["authority"]["imported_fingerprint"] = DIGEST_B if mutation == "missing_authority": @@ -168,7 +334,7 @@ def test_rejects_incomplete_or_unsafe_evidence(mutation): if mutation == "provider": value["host"]["provider"] = "local" if mutation == "capacity": - value["host"]["memory_bytes"] -= 1 + value["host"]["memory_bytes"] = 0 if mutation == "failed_result": value["result"] = "fail" value["first_failure"] = "generate" diff --git a/scripts/ci/test-validate-g500-ladder-qualification.py b/scripts/ci/test-validate-g500-ladder-qualification.py new file mode 100644 index 000000000..d2e158e11 --- /dev/null +++ b/scripts/ci/test-validate-g500-ladder-qualification.py @@ -0,0 +1,347 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +SCRIPT = Path(__file__).with_name("validate-g500-ladder-qualification.py") +SPEC = importlib.util.spec_from_file_location("ladder_qualification", SCRIPT) +assert SPEC and SPEC.loader +VALIDATOR = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(VALIDATOR) + +CATEGORIES = ( + ("canonical_node_topology", "storage_owned_snapshot"), + ("canonical_edge_topology", "storage_owned_snapshot"), + ("properties", "storage_owned_snapshot"), + ("uuid_surrogate_indexes", "storage_owned_snapshot"), + ("adjacency_csr", "storage_owned_snapshot"), + ("catalog_manifests", "storage_owned_snapshot"), + ("construction_staging_spill", "construction_receipts"), + ("portable_package", "exact_descriptor"), + ("clean_imported_project", "clean_import_snapshot"), +) +PHASES = ( + "append_merge", + "seal_authentication", + "shape_consume_reauthentication", + "encode_write_postwrite_authentication", + "publication_preauthentication", + "cas_install_read_write", + "hydration_verification", + "fsync_synchronization", + "recovery_reauthentication", +) + + +def rung(scale: int, live: int, unit: int) -> dict: + artifacts = [ + { + "category": category, + "logical_bytes": unit * (index + 1), + "allocated_bytes": unit * (index + 2), + "current_retained_bytes": unit * (index + 1), + "transient_peak_allocated_bytes": unit * 100 if index == 0 else unit, + "logical_references": index + 2, + "physical_objects": index + 1, + "source": source, + } + for index, (category, source) in enumerate(CATEGORIES) + ] + logical = sum(item["logical_bytes"] for item in artifacts) + allocated = sum(item["allocated_bytes"] for item in artifacts) + retained = sum(item["current_retained_bytes"] for item in artifacts) + selected_source = sum(item["allocated_bytes"] for item in artifacts[:6]) + source_project = selected_source + unit + # Independent union high-water observation; deliberately larger than any + # one category peak because categories coexist at lifecycle boundaries. + peak = sum(item["transient_peak_allocated_bytes"] for item in artifacts) + phases = [ + { + "phase": phase, + "applicable": True, + "read_bytes": unit, + "write_bytes": unit, + "read_calls": 1, + "write_calls": 1, + "object_count": 1, + "block_count": 1, + "fsync_calls": 1, + } + for phase in PHASES + ] + return { + "id": f"S{scale}", + "scale": scale, + "live_nodes": live // 16, + "live_edges": live, + "source_project_current_allocated_bytes": source_project, + "workspace_current_allocated_bytes": retained, + "artifacts": artifacts, + "phases": phases, + "totals": { + "logical_bytes": logical, + "allocated_bytes": allocated, + "current_retained_bytes": retained, + "transient_peak_allocated_bytes": peak, + "phase_read_bytes": unit * 9, + "phase_write_bytes": unit * 9, + "phase_read_calls": 9, + "phase_write_calls": 9, + "phase_object_count": 9, + "phase_block_count": 9, + "phase_fsync_calls": 9, + }, + "ratios": { + "canonical_node_bytes_per_live_node": { + "numerator_bytes": artifacts[0]["logical_bytes"], + "denominator_count": live // 16, + }, + "canonical_edge_bytes_per_live_edge": { + "numerator_bytes": artifacts[1]["logical_bytes"], + "denominator_count": live, + }, + "authoritative_project_bytes_per_live_edge": { + "numerator_bytes": source_project, + "denominator_count": live, + }, + "full_lifecycle_peak_bytes_per_live_edge": { + "numerator_bytes": peak, + "denominator_count": live, + }, + }, + } + + +def evidence() -> dict: + low = rung(20, (1 << 20) * 16, 1_000) + high = rung(22, (1 << 22) * 16, 4_000) + numerator = high["totals"]["transient_peak_allocated_bytes"] + denominator = high["live_edges"] + projected = VALIDATOR.ceil_ratio(numerator * VALIDATOR.S26_EDGES, denominator) + volume = 50_000_000_000 + return { + "schema": "graphforge-g500-ladder-qualification/3", + "rungs": [low, high], + "projection": { + "target": "S26", + "source_rungs": ["S20", "S22"], + "rate": { + "numerator_bytes": numerator, + "denominator_count": denominator, + }, + "projected_canonical_node_bytes": VALIDATOR.ceil_ratio( + high["artifacts"][0]["current_retained_bytes"] * VALIDATOR.S26_NODES, + high["live_nodes"], + ), + "projected_canonical_edge_bytes": VALIDATOR.ceil_ratio( + high["artifacts"][1]["current_retained_bytes"] * VALIDATOR.S26_EDGES, + high["live_edges"], + ), + "projected_lifecycle_peak_bytes": projected, + "volume_bytes": volume, + "reserved_headroom_bytes": 500_000_000, + "headroom_bytes": volume - projected, + "decision": "admit", + }, + } + + +def test_accepts_reconciled_adjacent_rungs_and_conservative_projection(): + VALIDATOR.validate(evidence()) + + +def test_accepts_closed_truthful_zero_io_phase(): + value = evidence() + phase = value["rungs"][0]["phases"][0] + for field in ( + "read_bytes", + "write_bytes", + "read_calls", + "write_calls", + "object_count", + "block_count", + "fsync_calls", + ): + value["rungs"][0]["totals"][f"phase_{field}"] -= phase[field] + phase[field] = 0 + phase["applicable"] = False + VALIDATOR.validate(value) + + +@pytest.mark.parametrize( + "mutation,match", + [ + ("missing_category", "schema violation"), + ("duplicate_category", "complete and unique"), + ("undeduplicated", "physical identities must be deduplicated"), + ("logical_total", "totals do not reconcile"), + ("allocated_total", "totals do not reconcile"), + ("denominator", "reproducible denominators"), + ("node_scale", "declared scale"), + ("edge_envelope", "Graph500 envelope"), + ("one_rung", "schema violation"), + ("nonadjacent", "ordered, and adjacent"), + ("understated_slope", "below an observed"), + ("projection", "not reproducible"), + ("headroom", "does not reconcile"), + ("unsafe_admit", "contradicts projected headroom"), + ("peak_below_artifact", "below a category peak"), + ("false_applicability", "applicability contradicts"), + ], +) +def test_rejects_goal_seeking_or_incomplete_evidence(mutation: str, match: str): + value = copy.deepcopy(evidence()) + if mutation == "missing_category": + value["rungs"][0]["artifacts"].pop() + elif mutation == "duplicate_category": + value["rungs"][0]["artifacts"][4] = copy.deepcopy(value["rungs"][0]["artifacts"][0]) + elif mutation == "undeduplicated": + value["rungs"][0]["artifacts"][0]["physical_objects"] = 3 + elif mutation == "logical_total": + value["rungs"][0]["totals"]["logical_bytes"] += 1 + elif mutation == "allocated_total": + value["rungs"][0]["totals"]["allocated_bytes"] += 1 + elif mutation == "denominator": + value["rungs"][0]["ratios"]["authoritative_project_bytes_per_live_edge"][ + "denominator_count" + ] += 1 + elif mutation == "node_scale": + value["rungs"][0]["live_nodes"] -= 1 + elif mutation == "edge_envelope": + value["rungs"][0]["live_edges"] = value["rungs"][0]["live_nodes"] * 16 + 1 + elif mutation == "one_rung": + value["rungs"].pop() + elif mutation == "nonadjacent": + value["rungs"][1]["id"], value["rungs"][1]["scale"] = "S24", 24 + elif mutation == "understated_slope": + value["projection"]["rate"]["numerator_bytes"] = 1 + elif mutation == "projection": + value["projection"]["projected_lifecycle_peak_bytes"] += 1 + elif mutation == "headroom": + value["projection"]["headroom_bytes"] += 1 + elif mutation == "unsafe_admit": + value["projection"]["reserved_headroom_bytes"] = value["projection"]["headroom_bytes"] + 1 + elif mutation == "peak_below_artifact": + value["rungs"][0]["totals"]["transient_peak_allocated_bytes"] = 0 + elif mutation == "false_applicability": + value["rungs"][0]["phases"][0]["applicable"] = False + with pytest.raises(VALIDATOR.EvidenceError, match=match): + VALIDATOR.validate(value) + + +def test_refuses_when_projection_does_not_leave_reserved_headroom(): + value = evidence() + value["projection"]["reserved_headroom_bytes"] = value["projection"]["headroom_bytes"] + 1 + value["projection"]["decision"] = "refuse" + VALIDATOR.validate(value) + + +def test_refuses_volume_overflow_even_with_zero_reserved_headroom(): + value = evidence() + value["projection"]["volume_bytes"] = value["projection"]["projected_lifecycle_peak_bytes"] - 1 + value["projection"]["reserved_headroom_bytes"] = 0 + value["projection"]["headroom_bytes"] = 0 + value["projection"]["decision"] = "refuse" + VALIDATOR.validate(value) + + +def test_canonical_projection_excludes_package_and_import_copies(): + value = evidence() + value["projection"]["projected_canonical_edge_bytes"] = VALIDATOR.ceil_ratio( + value["rungs"][-1]["totals"]["current_retained_bytes"] * VALIDATOR.S26_EDGES, + value["rungs"][-1]["live_edges"], + ) + with pytest.raises(VALIDATOR.EvidenceError, match="canonical edge projection"): + VALIDATOR.validate(value) + + +def certification_document(scale: int, edges: int, unit: int) -> dict: + phase_values = { + phase: { + "read_bytes": unit, + "write_bytes": unit, + "read_calls": 1, + "write_calls": 1, + "object_count": 1, + "block_count": 1, + "fsync_calls": 1, + } + for phase in PHASES + } + categories = { + key: { + "logical_references": index + 2, + "logical_bytes": unit * (index + 1), + "physical_objects": index + 1, + "physical_logical_bytes": unit * (index + 1), + "allocated_bytes": unit * (index + 2), + } + for index, key in enumerate( + ( + "topology_nodes", + "topology_edges", + "properties", + "uuid_and_surrogates", + "adjacency", + "catalog_and_manifests", + ) + ) + } + descriptor = { + "logical_bytes": unit, + "allocated_bytes": unit * 2, + "logical_references": 1, + "physical_objects": 1, + } + return { + "run": {"scale": scale}, + "counts": {"source_nodes": 1 << scale, "source_edges": edges}, + "envelope": { + "peak_disk_source": "storage_owned_active_identity_union", + "peak_disk_bytes": unit * 100, + }, + "storage_attribution": { + "source": { + "categories": categories, + "allocated_bytes": sum(item["allocated_bytes"] for item in categories.values()), + }, + "source_project_current_allocated_bytes": unit * 28, + "portable_package": descriptor, + "clean_import": descriptor, + "construction": { + "storage_current": {"construction_staging": descriptor}, + "storage_transient_peak_total_allocated_bytes": unit * 5, + }, + "application_io_phases": {"phases": phase_values}, + "workspace_current_allocated_bytes": unit * 30, + }, + } + + +def test_real_certification_companion_builds_then_validates(tmp_path: Path): + low = tmp_path / "s20.json" + high = tmp_path / "s22.json" + output = tmp_path / "qualification.json" + low.write_text(json.dumps(certification_document(20, 1 << 24, 1_000))) + high.write_text(json.dumps(certification_document(22, 1 << 26, 4_000))) + subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name("build-g500-ladder-qualification.py")), + str(low), + str(high), + str(output), + "--volume-bytes", + "500000000000", + "--reserved-headroom-bytes", + "1000000000", + ], + check=True, + ) + VALIDATOR.validate(json.loads(output.read_text())) diff --git a/scripts/ci/validate-g500-certification.py b/scripts/ci/validate-g500-certification.py index 26a357ee4..783472551 100644 --- a/scripts/ci/validate-g500-certification.py +++ b/scripts/ci/validate-g500-certification.py @@ -32,8 +32,15 @@ "drill_resource_limit", "drill_interrupted_finalization", ) -FORBIDDEN_KEY = re.compile(r"(secret|credential|token|password|host_path|absolute_path)", re.I) +FORBIDDEN_KEY = re.compile( + r"(secret|credential|token|password|host_path|absolute_path|machine[_-]?id|volume[_-]?id|provider_resource_id)", + re.I, +) ABSOLUTE_PATH = re.compile(r"(?:^|[\s=:])(?:/|[A-Za-z]:[\\/])") +RAW_UUID = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + re.I, +) ROOT = Path(__file__).resolve().parents[2] PROFILE = ROOT / "crates/graphforge-api/tests/fixtures/scale_g500_certification.v1.json" SCHEMA = ROOT / "docs/development/evidence/g500-certification.schema.json" @@ -79,8 +86,11 @@ def reject_sensitive(value: Any, trail: str = "$") -> None: elif isinstance(value, list): for index, child in enumerate(value): reject_sensitive(child, f"{trail}[{index}]") - elif isinstance(value, str) and ABSOLUTE_PATH.search(value): - raise EvidenceError(f"absolute host path at {trail}") + elif isinstance(value, str): + if ABSOLUTE_PATH.search(value): + raise EvidenceError(f"absolute host path at {trail}") + if RAW_UUID.fullmatch(value): + raise EvidenceError(f"raw UUID at {trail}") def validate(evidence: dict[str, Any], expected_sha: str | None) -> None: @@ -97,9 +107,13 @@ def validate(evidence: dict[str, Any], expected_sha: str | None) -> None: expected_profile = "sha256:" + hashlib.sha256(PROFILE.read_bytes()).hexdigest() if evidence.get("profile_sha256") != expected_profile: raise EvidenceError("evidence profile does not match the committed certification profile") + run = evidence.get("run", {}) + scale = run.get("scale") + if scale not in (20, 22, 24, 26): + raise EvidenceError("run scale is not a supported qualification rung") expected_run = { "command": RUN_COMMAND, - "scale": 26, + "scale": scale, "edgefactor": 16, "seed": 1, "directionality": "undirected", @@ -130,16 +144,37 @@ def validate(evidence: dict[str, Any], expected_sha: str | None) -> None: raise EvidenceError("counts must be non-negative integers") if raw != live + loops + dupes: raise EvidenceError("generator counts do not reconcile") - if live < 1_000_000_000: - raise EvidenceError("certification requires at least one billion live edges") + if scale == 26 and live < 1_000_000_000: + raise EvidenceError("S26 certification requires at least one billion live edges") if any(counts.get(key) != live for key in ("source_edges", "imported_edges")): raise EvidenceError("source/imported edge counts differ") if counts.get("source_nodes") != counts.get("imported_nodes"): raise EvidenceError("source/imported node counts differ") + if counts.get("source_nodes") != 1 << scale: + raise EvidenceError("source/imported node count does not match the declared scale") + + storage = evidence.get("storage_attribution", {}) + selected_source = storage.get("source", {}).get("allocated_bytes") + source_project = storage.get("source_project_current_allocated_bytes") + workspace = storage.get("workspace_current_allocated_bytes") + peak = evidence.get("envelope", {}).get("peak_disk_bytes") + if not all( + isinstance(value, int) for value in (selected_source, source_project, workspace, peak) + ): + raise EvidenceError("storage union numerators must be integers") + if not selected_source <= source_project <= workspace <= peak: + raise EvidenceError( + "selected source, source project, workspace, and peak unions do not reconcile" + ) identities = evidence.get("identities", {}) - if identities.get("source_generation") == identities.get("imported_generation"): - raise EvidenceError("source and imported generations must be distinct") + for proof in ( + "source_export_generation_authenticated", + "import_receipt_reopen_authenticated", + "source_import_generations_distinct", + ): + if identities.get(proof) is not True: + raise EvidenceError(f"generation proof is not authenticated: {proof}") if len({identities.get("package"), identities.get("transport")}) != 2: raise EvidenceError("semantic package and transport identities must be distinct") package = evidence.get("package", {}) @@ -233,11 +268,12 @@ def validate(evidence: dict[str, Any], expected_sha: str | None) -> None: raise EvidenceError("certification OS image contains unsupported characters") if re.fullmatch(r"[0-9A-Za-z._+-]+", str(host.get("kernel", ""))) is None: raise EvidenceError("host kernel release is malformed") - if ( - host.get("memory_bytes", 0) < 137_438_953_472 - or host.get("nvme_bytes", 0) < 1_099_511_627_776 - ): - raise EvidenceError("host does not meet declared capacity") + memory_bytes = host.get("memory_bytes", 0) + nvme_bytes = host.get("nvme_bytes", 0) + if memory_bytes < envelope.get("peak_rss_bytes", 0): + raise EvidenceError("observed RSS exceeds declared host memory") + if nvme_bytes < envelope.get("peak_disk_bytes", 0): + raise EvidenceError("observed storage peak exceeds declared host capacity") if evidence.get("result") != "pass" or evidence.get("first_failure") is not None: raise EvidenceError("certification evidence is not a pass") diff --git a/scripts/ci/validate-g500-ladder-qualification.py b/scripts/ci/validate-g500-ladder-qualification.py new file mode 100644 index 000000000..3c286b656 --- /dev/null +++ b/scripts/ci/validate-g500-ladder-qualification.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Fail-closed semantic validator for #951 disk attribution and S26 projection.""" + +from __future__ import annotations + +import argparse +from itertools import pairwise +import json +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError, ValidationError + +ROOT = Path(__file__).resolve().parents[2] +SCHEMA = ROOT / "docs/development/evidence/g500-ladder-qualification.schema.json" +REQUIRED_CATEGORIES = { + "canonical_node_topology", + "canonical_edge_topology", + "properties", + "uuid_surrogate_indexes", + "adjacency_csr", + "catalog_manifests", + "construction_staging_spill", + "portable_package", + "clean_imported_project", +} +REQUIRED_PHASES = { + "append_merge", + "seal_authentication", + "shape_consume_reauthentication", + "encode_write_postwrite_authentication", + "publication_preauthentication", + "cas_install_read_write", + "hydration_verification", + "fsync_synchronization", + "recovery_reauthentication", +} +S26_EDGES = 1 << 30 # SCALE=26, edgefactor=16 raw target; conservative live denominator. +S26_NODES = 1 << 26 + + +class EvidenceError(ValueError): + pass + + +def ceil_ratio(numerator: int, denominator: int) -> int: + return (numerator + denominator - 1) // denominator + + +def validate_schema(evidence: dict[str, Any]) -> None: + try: + contract = json.loads(SCHEMA.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(contract) + Draft202012Validator(contract).validate(evidence) + except (OSError, json.JSONDecodeError, SchemaError) as error: + raise EvidenceError(f"committed schema is invalid: {error}") from error + except ValidationError as error: + location = ".".join(str(part) for part in error.absolute_path) or "$" + raise EvidenceError(f"schema violation at {location}: {error.message}") from error + + +def validate(evidence: dict[str, Any]) -> None: + validate_schema(evidence) + rungs = evidence["rungs"] + if len(rungs) < 2: + raise EvidenceError("at least two adjacent observations are required") + scales = [rung["scale"] for rung in rungs] + if scales != sorted(set(scales)) or any(b - a != 2 for a, b in pairwise(scales)): + raise EvidenceError("rungs must be unique, ordered, and adjacent") + + for rung in rungs: + if rung["id"] != f"S{rung['scale']}": + raise EvidenceError("rung id and scale disagree") + categories = [artifact["category"] for artifact in rung["artifacts"]] + if set(categories) != REQUIRED_CATEGORIES or len(categories) != len(set(categories)): + raise EvidenceError("artifact categories must be complete and unique") + phases = rung["phases"] + phase_names = [phase["phase"] for phase in phases] + if set(phase_names) != REQUIRED_PHASES or len(phase_names) != len(set(phase_names)): + raise EvidenceError("application I/O phases must be complete and unique") + phase_fields = ( + "read_bytes", + "write_bytes", + "read_calls", + "write_calls", + "object_count", + "block_count", + "fsync_calls", + ) + for phase in phases: + observed = any(phase[field] != 0 for field in phase_fields) + if phase["applicable"] != observed: + raise EvidenceError("phase applicability contradicts source-owned counters") + if (phase["read_bytes"] == 0) != (phase["read_calls"] == 0): + raise EvidenceError("phase read bytes and calls disagree") + if (phase["write_bytes"] == 0) != (phase["write_calls"] == 0): + raise EvidenceError("phase write bytes and calls disagree") + if any( + artifact["physical_objects"] > artifact["logical_references"] + for artifact in rung["artifacts"] + ): + raise EvidenceError("physical identities must be deduplicated from logical references") + logical = sum(artifact["logical_bytes"] for artifact in rung["artifacts"]) + allocated = sum(artifact["allocated_bytes"] for artifact in rung["artifacts"]) + retained_views = sum(artifact["current_retained_bytes"] for artifact in rung["artifacts"]) + retained = rung["totals"]["current_retained_bytes"] + if retained != rung["workspace_current_allocated_bytes"]: + raise EvidenceError("workspace numerator disagrees with retained identity union") + # Category peaks are diagnostics, not a total: categories coexist. + # The total is an independently observed phase-boundary union high-water + # mark and must not be reconstructed as max(category). + transient_peak = rung["totals"]["transient_peak_allocated_bytes"] + phase_totals = { + f"phase_{field}": sum(phase[field] for phase in phases) for field in phase_fields + } + expected_totals = {"logical_bytes": logical, "allocated_bytes": allocated, **phase_totals} + if {key: rung["totals"][key] for key in expected_totals} != expected_totals: + raise EvidenceError("artifact or phase totals do not reconcile") + if transient_peak < max( + artifact["transient_peak_allocated_bytes"] for artifact in rung["artifacts"] + ): + raise EvidenceError("lifecycle peak is below a category peak") + if retained > retained_views or retained < max( + artifact["current_retained_bytes"] for artifact in rung["artifacts"] + ): + raise EvidenceError("native retained union is inconsistent with owner views") + if any( + item["current_retained_bytes"] > item["allocated_bytes"] for item in rung["artifacts"] + ): + raise EvidenceError("retained allocation exceeds category allocation") + if transient_peak < retained: + raise EvidenceError("lifecycle peak is below current retained allocation") + source_project = rung["source_project_current_allocated_bytes"] + selected_source = sum( + item["allocated_bytes"] + for item in rung["artifacts"] + if item["source"] == "storage_owned_snapshot" + ) + if source_project < selected_source: + raise EvidenceError("source project union is below its selected generation") + if source_project > retained: + raise EvidenceError("source project union exceeds the workspace union") + live, nodes = rung["live_edges"], rung["live_nodes"] + if nodes != 1 << rung["scale"]: + raise EvidenceError("live node denominator disagrees with declared scale") + if not 0 < live <= nodes * 16: + raise EvidenceError("live edge denominator exceeds the Graph500 envelope") + by_category = {item["category"]: item for item in rung["artifacts"]} + expected = { + "canonical_node_bytes_per_live_node": { + "numerator_bytes": by_category["canonical_node_topology"]["logical_bytes"], + "denominator_count": nodes, + }, + "canonical_edge_bytes_per_live_edge": { + "numerator_bytes": by_category["canonical_edge_topology"]["logical_bytes"], + "denominator_count": live, + }, + "authoritative_project_bytes_per_live_edge": { + "numerator_bytes": source_project, + "denominator_count": live, + }, + "full_lifecycle_peak_bytes_per_live_edge": { + "numerator_bytes": transient_peak, + "denominator_count": live, + }, + } + if rung["ratios"] != expected: + raise EvidenceError("ratios must preserve exact reproducible denominators") + + rate = evidence["projection"]["rate"] + rn, rd = rate["numerator_bytes"], rate["denominator_count"] + for low, high in pairwise(rungs): + delta_edges = high["live_edges"] - low["live_edges"] + delta_bytes = ( + high["totals"]["transient_peak_allocated_bytes"] + - low["totals"]["transient_peak_allocated_bytes"] + ) + if delta_edges <= 0: + raise EvidenceError("live-edge denominator must increase across adjacent rungs") + if delta_bytes > 0 and rn * delta_edges < delta_bytes * rd: + raise EvidenceError("projection rate is below an observed adjacent-rung slope") + if rn * high["live_edges"] < high["totals"]["transient_peak_allocated_bytes"] * rd: + raise EvidenceError("projection rate is below the latest observed peak ratio") + + projected = ceil_ratio(rn * S26_EDGES, rd) + projection = evidence["projection"] + if projection["source_rungs"] != [rungs[-2]["id"], rungs[-1]["id"]]: + raise EvidenceError("projection must cite the newest adjacent source rungs") + if projection["projected_lifecycle_peak_bytes"] != projected: + raise EvidenceError("S26 projected peak is not reproducible from the declared rate") + latest_categories = {item["category"]: item for item in rungs[-1]["artifacts"]} + canonical_node_projected = ceil_ratio( + latest_categories["canonical_node_topology"]["current_retained_bytes"] * S26_NODES, + rungs[-1]["live_nodes"], + ) + canonical_edge_projected = ceil_ratio( + latest_categories["canonical_edge_topology"]["current_retained_bytes"] * S26_EDGES, + rungs[-1]["live_edges"], + ) + if projection["projected_canonical_node_bytes"] != canonical_node_projected: + raise EvidenceError("S26 canonical node projection is not reproducible") + if projection["projected_canonical_edge_bytes"] != canonical_edge_projected: + raise EvidenceError("S26 canonical edge projection is not reproducible") + if projected > projection["volume_bytes"]: + expected_headroom = 0 + else: + expected_headroom = projection["volume_bytes"] - projected + if projection["headroom_bytes"] != expected_headroom: + raise EvidenceError("headroom does not reconcile") + expected_decision = "refuse" + if ( + projected <= projection["volume_bytes"] + and expected_headroom >= projection["reserved_headroom_bytes"] + ): + expected_decision = "admit" + if projection["decision"] != expected_decision: + raise EvidenceError("S26 admission decision contradicts projected headroom") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("evidence", type=Path) + args = parser.parse_args() + try: + value = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise EvidenceError("evidence root must be an object") + validate(value) + except (OSError, json.JSONDecodeError, EvidenceError) as error: + raise SystemExit(str(error)) from error + + +if __name__ == "__main__": + main() diff --git a/tests/contracts/non-cypher-rust-surface.json b/tests/contracts/non-cypher-rust-surface.json index a3f59e7f8..5aa8a51f0 100644 --- a/tests/contracts/non-cypher-rust-surface.json +++ b/tests/contracts/non-cypher-rust-surface.json @@ -1,7 +1,7 @@ { "contract_version": 1, "scope": "Rust non-Cypher public release surface", - "public_method_digest": "71c92e47b6e43da553be7288724d522e303da5d4677e60f51e7ea8d9f8a41b99", + "public_method_digest": "8e3a0711619a5e50231bf510a76328ea44b7706dfd28c524b760c6564b805bc3", "method_policy": { "receiver_defaults": { "GraphForge": "release-tested", @@ -264,10 +264,15 @@ "GraphForge.publish_bulk_edges", "GraphForge.publish_bulk_nodes", "GraphForge.relationship_types", + "GraphForge.storage_attribution", "GraphForge.workspace_configuration", "GraphForge.workspace_ontology" ], "test_refs": [ + { + "path": "crates/graphforge-api/tests/scale_g500_ladder.rs", + "symbol": "public_storage_attribution_is_generation_bound_and_fully_classified" + }, { "path": "crates/graphforge-api/tests/public_lifecycle_conformance.rs", "symbol": "persisted_construction_reopens_with_exact_uuid_properties_and_order" diff --git a/tools/bazel/drift/cargo_feature_fingerprint.json b/tools/bazel/drift/cargo_feature_fingerprint.json index 28fcf6833..2558e9399 100644 --- a/tools/bazel/drift/cargo_feature_fingerprint.json +++ b/tools/bazel/drift/cargo_feature_fingerprint.json @@ -1,6 +1,6 @@ { "schema": "graphforge.cargo-feature-fingerprint.v1", - "sha256": "faf566b1023d302f374da985eedf1d2faa6e580fb9caa5dccc17840db4c50023", + "sha256": "7bff132d4c674e4bdc233fd065ffb378e9512889b36ca1316fdd40033b9a4e13", "entries": [ { "name": "graphforge-api", @@ -90,6 +90,15 @@ "kind": null, "target": null }, + { + "name": "graphforge-filesystem", + "req": "*", + "features": [], + "optional": false, + "uses_default_features": true, + "kind": "dev", + "target": null + }, { "name": "graphforge-io", "req": "^0.5.2", @@ -1105,7 +1114,8 @@ "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", - "Win32_System_IO" + "Win32_System_IO", + "Win32_System_Ioctl" ], "optional": false, "uses_default_features": true,