diff --git a/benchmarks/Cargo.lock b/benchmarks/Cargo.lock index ba091805..6c74992f 100644 --- a/benchmarks/Cargo.lock +++ b/benchmarks/Cargo.lock @@ -2,6 +2,110 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "graphforge-benchmark-certify" +version = "0.0.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "graphforge-benchmark-smoke" version = "0.0.0" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 89dd5412..638d5928 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["runners/smoke"] +members = ["runners/certify", "runners/smoke"] resolver = "2" [workspace.package] @@ -7,3 +7,7 @@ edition = "2024" license = "Apache-2.0" publish = false rust-version = "1.85" + +[workspace.dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/benchmarks/README.md b/benchmarks/README.md index e98b5bd2..9d39797f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -26,6 +26,56 @@ belongs to issue #959, after parity is proven. - `tests/` — no-cost workspace and fixture-discovery tests. - `runners/` — unpublished Rust benchmark runners. +## Public-interface certification runner + +`runners/certify` owns the admission through reopen-proof lifecycle. A profile +declares a typed benchmark-owned generator action for `generate` and one +ordinary `gf` public command action for every product phase, in order. Runtime +validation binds each product phase to its real command family and rejects +global help/version no-ops. The runner does not link product crates, call +storage internals, provision a host, or enforce CPU, memory, disk, or time +limits. BenchExec or another outer orchestrator owns deadlines and may terminate +the runner; the certification binary does not silently convert a policy timeout +into product evidence. + +```bash +cargo run --locked --manifest-path benchmarks/Cargo.toml \ + --bin graphforge-benchmark-certify -- \ + run PROFILE.json benchmarks/outputs/evidence.json +``` + +Tiny in-process fixtures exercise every phase and first-failure behavior without +opening a project. Graph500 profiles supply the real generate, ingest, recount, +query, portable export/verify/import, and reopen commands in #956; a placeholder +profile that merely labels no-op commands as lifecycle work is intentionally not +shipped. + +Evidence contains only phase names, typed pass/fail state, exit codes, elapsed +milliseconds, and observed peak RSS bytes. Command arguments and child output +are intentionally excluded so graph contents, credentials, and sensitive paths +cannot be emitted. Missing RSS is represented as `null`; this runner records +resource evidence but never enforces a resource policy. Certification stops at +the first failed phase. One sanitized JSON phase event is written to standard +output as each public command finishes; the final typed evidence document is +written only to the requested output file. + +Phase RSS is an observation, not a memory budget. Ladder orchestration must +compare the same phase across scales and treat sustained edge-count-linear RSS +growth as an architectural failure signal. This runner does not infer that +cross-profile trend or convert the M5 certification ceiling into a requirement. + +Legacy JSON can be converted into the same sanitized contract: + +```bash +cargo run --locked --manifest-path benchmarks/Cargo.toml \ + --bin graphforge-benchmark-certify -- \ + normalize legacy-evidence.json benchmarks/outputs/normalized.json +``` + +The accepted legacy shape is `{profile, phases}` with each phase containing +`name`, `ok`, `duration_secs`, optional `max_rss_kib`, and optional +`exit_code`. Normalization is fail-closed and stops after the first failure. + Generated datasets, credentials, execution outputs, and local environments are ignored. Fly execution is forbidden until the complete benchmark stack is merged and has passed local and hosted qualification. diff --git a/benchmarks/runners/certify/Cargo.toml b/benchmarks/runners/certify/Cargo.toml new file mode 100644 index 00000000..073dd1ed --- /dev/null +++ b/benchmarks/runners/certify/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "graphforge-benchmark-certify" +version = "0.0.0" +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/benchmarks/runners/certify/src/lib.rs b/benchmarks/runners/certify/src/lib.rs new file mode 100644 index 00000000..d608427e --- /dev/null +++ b/benchmarks/runners/certify/src/lib.rs @@ -0,0 +1,686 @@ +#![forbid(unsafe_code)] + +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::fs; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +pub const EVIDENCE_SCHEMA: &str = "graphforge-public-certification/1"; +pub const PHASE_EVENT_SCHEMA: &str = "graphforge-public-certification-phase-event/1"; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Phase { + Admission, + Generate, + Ingest, + Reopen, + Recount, + Query, + Export, + Verify, + CleanImport, + ReopenProof, +} + +impl Phase { + pub const ALL: [Self; 10] = [ + Self::Admission, + Self::Generate, + Self::Ingest, + Self::Reopen, + Self::Recount, + Self::Query, + Self::Export, + Self::Verify, + Self::CleanImport, + Self::ReopenProof, + ]; +} + +impl fmt::Display for Phase { + fn fmt(&self, output: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = serde_json::to_value(self).map_err(|_| fmt::Error)?; + output.write_str(value.as_str().ok_or(fmt::Error)?) + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Profile { + pub schema: String, + pub id: String, + pub executable: String, + pub phases: Vec, +} + +impl Profile { + pub fn validate(&self) -> Result<(), RunnerError> { + if self.schema != "graphforge-public-certification-profile/1" { + return Err(RunnerError::Profile("unsupported profile schema")); + } + if !is_safe_token(&self.id) { + return Err(RunnerError::Profile("profile id must be a safe token")); + } + if !is_graphforge_executable(&self.executable) { + return Err(RunnerError::Profile( + "executable must resolve to the public gf command", + )); + } + if self.phases.len() != Phase::ALL.len() + || self + .phases + .iter() + .map(|command| command.phase) + .ne(Phase::ALL) + { + return Err(RunnerError::Profile( + "profile must declare every phase once in lifecycle order", + )); + } + if self + .phases + .iter() + .any(|command| !action_matches_phase(command)) + { + return Err(RunnerError::Profile( + "phase action must select the matching benchmark or public gf operation", + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct PhaseCommand { + pub phase: Phase, + pub action: PhaseAction, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[serde(tag = "interface", rename_all = "snake_case", deny_unknown_fields)] +pub enum PhaseAction { + BenchmarkGenerator { + identity: String, + executable: String, + args: Vec, + }, + GraphForgeCli { + args: Vec, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Execution { + pub exit_code: Option, + pub duration_ms: u64, + pub peak_rss_bytes: Option, +} + +pub trait PhaseExecutor { + fn execute(&mut self, profile: &Profile, command: &PhaseCommand) -> Result; +} + +#[derive(Default)] +pub struct PublicProcessExecutor; + +impl PhaseExecutor for PublicProcessExecutor { + fn execute(&mut self, profile: &Profile, command: &PhaseCommand) -> Result { + let (executable, args) = match &command.action { + PhaseAction::BenchmarkGenerator { + executable, args, .. + } => (executable.as_str(), args.as_slice()), + PhaseAction::GraphForgeCli { args } => (profile.executable.as_str(), args.as_slice()), + }; + let started = Instant::now(); + let mut child = Command::new(executable) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|_| "public command could not start".to_owned())?; + let mut peak_rss_bytes = None; + loop { + peak_rss_bytes = max_optional(peak_rss_bytes, resident_bytes(child.id())); + if let Some(status) = child + .try_wait() + .map_err(|_| "public command wait failed".to_owned())? + { + return Ok(Execution { + exit_code: status.code(), + duration_ms: millis(started.elapsed()), + peak_rss_bytes, + }); + } + thread::sleep(Duration::from_millis(10)); + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OutcomeStatus { + Passed, + Failed, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum FailureKind { + CommandFailed, + CommandUnavailable, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PhaseOutcome { + pub phase: Phase, + pub status: OutcomeStatus, + pub duration_ms: u64, + pub peak_rss_bytes: Option, + pub exit_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub failure: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Evidence { + pub schema: String, + pub profile_id: String, + pub status: OutcomeStatus, + pub phases: Vec, + pub failed_phase: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PhaseEvent { + pub schema: &'static str, + pub profile_id: String, + pub outcome: PhaseOutcome, +} + +pub fn certify( + profile: &Profile, + executor: &mut dyn PhaseExecutor, +) -> Result { + certify_with_events(profile, executor, |_| Ok(())) +} + +pub fn certify_with_events( + profile: &Profile, + executor: &mut dyn PhaseExecutor, + mut emit: impl FnMut(&PhaseEvent) -> Result<(), RunnerError>, +) -> Result { + profile.validate()?; + let mut phases = Vec::with_capacity(Phase::ALL.len()); + for command in &profile.phases { + let outcome = match executor.execute(profile, command) { + Ok(execution) => { + let passed = execution.exit_code == Some(0); + PhaseOutcome { + phase: command.phase, + status: if passed { + OutcomeStatus::Passed + } else { + OutcomeStatus::Failed + }, + duration_ms: execution.duration_ms, + peak_rss_bytes: execution.peak_rss_bytes, + exit_code: execution.exit_code, + failure: (!passed).then_some(FailureKind::CommandFailed), + } + } + Err(_) => PhaseOutcome { + phase: command.phase, + status: OutcomeStatus::Failed, + duration_ms: 0, + peak_rss_bytes: None, + exit_code: None, + failure: Some(FailureKind::CommandUnavailable), + }, + }; + let failed = outcome.status == OutcomeStatus::Failed; + emit(&PhaseEvent { + schema: PHASE_EVENT_SCHEMA, + profile_id: profile.id.clone(), + outcome: outcome.clone(), + })?; + phases.push(outcome); + if failed { + break; + } + } + let failed_phase = phases + .iter() + .find(|outcome| outcome.status == OutcomeStatus::Failed) + .map(|outcome| outcome.phase); + Ok(Evidence { + schema: EVIDENCE_SCHEMA.to_owned(), + profile_id: profile.id.clone(), + status: if failed_phase.is_some() { + OutcomeStatus::Failed + } else { + OutcomeStatus::Passed + }, + phases, + failed_phase, + }) +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum EvidenceInput { + Current(Evidence), + Legacy(LegacyEvidence), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyEvidence { + profile: String, + phases: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyPhase { + name: Phase, + ok: bool, + duration_secs: f64, + max_rss_kib: Option, + exit_code: Option, +} + +pub fn normalize_evidence(input: &[u8]) -> Result { + match serde_json::from_slice(input).map_err(|_| RunnerError::Legacy)? { + EvidenceInput::Current(evidence) => validate_evidence(evidence), + EvidenceInput::Legacy(legacy) => { + if !is_safe_token(&legacy.profile) || legacy.phases.is_empty() { + return Err(RunnerError::Legacy); + } + let mut phases = Vec::with_capacity(legacy.phases.len()); + for phase in legacy.phases { + if !phase.duration_secs.is_finite() || phase.duration_secs < 0.0 { + return Err(RunnerError::Legacy); + } + phases.push(PhaseOutcome { + phase: phase.name, + status: if phase.ok { + OutcomeStatus::Passed + } else { + OutcomeStatus::Failed + }, + duration_ms: (phase.duration_secs * 1_000.0).round() as u64, + peak_rss_bytes: phase.max_rss_kib.and_then(|value| value.checked_mul(1_024)), + exit_code: phase.exit_code, + failure: (!phase.ok).then_some(FailureKind::CommandFailed), + }); + if !phase.ok { + break; + } + } + let failed_phase = phases + .iter() + .find(|outcome| outcome.status == OutcomeStatus::Failed) + .map(|outcome| outcome.phase); + validate_evidence(Evidence { + schema: EVIDENCE_SCHEMA.to_owned(), + profile_id: legacy.profile, + status: if failed_phase.is_some() { + OutcomeStatus::Failed + } else { + OutcomeStatus::Passed + }, + phases, + failed_phase, + }) + } + } +} + +fn validate_evidence(evidence: Evidence) -> Result { + if evidence.schema != EVIDENCE_SCHEMA + || !is_safe_token(&evidence.profile_id) + || evidence.phases.is_empty() + || evidence + .phases + .iter() + .map(|outcome| outcome.phase) + .ne(Phase::ALL.into_iter().take(evidence.phases.len())) + { + return Err(RunnerError::Legacy); + } + let observed_failure = evidence + .phases + .iter() + .find(|outcome| outcome.status == OutcomeStatus::Failed) + .map(|outcome| outcome.phase); + let statuses_are_consistent = evidence.phases.iter().enumerate().all(|(index, outcome)| { + let failed = outcome.status == OutcomeStatus::Failed; + failed == outcome.failure.is_some() && (!failed || index + 1 == evidence.phases.len()) + }); + if observed_failure != evidence.failed_phase + || (observed_failure.is_some()) != (evidence.status == OutcomeStatus::Failed) + || !statuses_are_consistent + { + return Err(RunnerError::Legacy); + } + Ok(evidence) +} + +pub fn read_profile(path: &Path) -> Result { + let input = fs::read(path).map_err(|_| RunnerError::Io)?; + serde_json::from_slice(&input).map_err(|_| RunnerError::Profile("invalid profile JSON")) +} + +pub fn write_evidence(path: &Path, evidence: &Evidence) -> Result<(), RunnerError> { + let encoded = serde_json::to_vec_pretty(evidence).map_err(|_| RunnerError::Io)?; + fs::write(path, encoded).map_err(|_| RunnerError::Io) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RunnerError { + Io, + Legacy, + Profile(&'static str), +} + +fn is_safe_token(value: &str) -> bool { + !value.is_empty() + && value.len() <= 80 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) +} + +fn is_graphforge_executable(value: &str) -> bool { + !value.contains('\0') + && Path::new(value) + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| matches!(name, "gf" | "gf.exe")) +} + +fn action_matches_phase(command: &PhaseCommand) -> bool { + if let PhaseAction::BenchmarkGenerator { + identity, + executable, + args, + } = &command.action + { + return command.phase == Phase::Generate + && is_safe_token(identity) + && !executable.is_empty() + && !executable.contains('\0') + && args.iter().all(|argument| !argument.contains('\0')); + } + let PhaseAction::GraphForgeCli { args } = &command.action else { + return false; + }; + if command.phase == Phase::Generate || args.iter().any(|argument| argument.contains('\0')) { + return false; + } + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + if args + .iter() + .any(|argument| matches!(*argument, "--version" | "--help")) + { + return false; + } + match command.phase { + Phase::Admission => args.as_slice() == ["--info"], + Phase::Generate => false, + Phase::Ingest => contains_command(&args, &["import-session"]), + Phase::Reopen => contains_command(&args, &["recovery"]), + Phase::Recount | Phase::Query | Phase::ReopenProof => contains_command(&args, &["query"]), + Phase::Export => contains_command(&args, &["portable", "export"]), + Phase::Verify => contains_command(&args, &["portable", "verify"]), + Phase::CleanImport => contains_command(&args, &["portable", "import"]), + } +} + +fn contains_command(arguments: &[&str], command: &[&str]) -> bool { + arguments + .windows(command.len()) + .any(|window| window == command) +} + +fn millis(duration: Duration) -> u64 { + duration.as_millis().try_into().unwrap_or(u64::MAX) +} + +fn max_optional(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.max(right)), + (left, right) => left.or(right), + } +} + +#[cfg(target_os = "linux")] +fn resident_bytes(pid: u32) -> Option { + let status = fs::read_to_string(format!("/proc/{pid}/status")).ok()?; + let rss_kib = status + .lines() + .find_map(|line| line.strip_prefix("VmHWM:"))? + .split_ascii_whitespace() + .next()? + .parse::() + .ok()?; + rss_kib.checked_mul(1_024) +} + +#[cfg(not(target_os = "linux"))] +fn resident_bytes(_pid: u32) -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + + struct FakeExecutor { + executions: VecDeque>, + calls: Vec, + } + + impl PhaseExecutor for FakeExecutor { + fn execute( + &mut self, + _profile: &Profile, + command: &PhaseCommand, + ) -> Result { + self.calls.push(command.phase); + self.executions.pop_front().expect("fixture execution") + } + } + + fn tiny_profile() -> Profile { + Profile { + schema: "graphforge-public-certification-profile/1".to_owned(), + id: "tiny-public".to_owned(), + executable: "gf".to_owned(), + phases: Phase::ALL + .into_iter() + .map(|phase| PhaseCommand { + phase, + action: if phase == Phase::Generate { + PhaseAction::BenchmarkGenerator { + identity: "tiny-generator-v1".to_owned(), + executable: "tiny-generator".to_owned(), + args: vec!["--scale".to_owned(), "tiny".to_owned()], + } + } else { + PhaseAction::GraphForgeCli { + args: tiny_args(phase), + } + }, + }) + .collect(), + } + } + + fn tiny_args(phase: Phase) -> Vec { + let values: &[&str] = match phase { + Phase::Admission => &["--info"], + Phase::Generate => unreachable!("generate uses the benchmark-owned typed action"), + Phase::Ingest => &[ + "--project", + "generated/tiny-source", + "import-session", + "open", + ], + Phase::Reopen => &["--project", "generated/tiny-source", "recovery"], + Phase::Recount => &[ + "--project", + "generated/tiny-source", + "query", + "--cypher", + "MATCH (n) RETURN count(n)", + "--output", + "generated/recount.arrow", + ], + Phase::Query => &[ + "--project", + "generated/tiny-source", + "query", + "--cypher", + "MATCH (n) RETURN n.id", + "--output", + "generated/query.arrow", + ], + Phase::Export => &[ + "--project", + "generated/tiny-source", + "portable", + "export", + "--current", + "--output", + "generated/tiny-portable", + ], + Phase::Verify => &["portable", "verify", "--input", "generated/tiny-portable"], + Phase::CleanImport => &[ + "--project", + "generated/tiny-import", + "portable", + "import", + "--input", + "generated/tiny-portable", + "--idempotency-key", + "00000000-0000-4000-8000-000000000001", + ], + Phase::ReopenProof => &[ + "--project", + "generated/tiny-import", + "query", + "--cypher", + "MATCH (n) RETURN count(n)", + "--output", + "generated/reopen-proof.arrow", + ], + }; + values.iter().map(|value| (*value).to_owned()).collect() + } + + fn passed_execution(index: u64) -> Result { + Ok(Execution { + exit_code: Some(0), + duration_ms: index + 1, + peak_rss_bytes: Some((index + 1) * 1_024), + }) + } + + #[test] + fn tiny_fixture_proves_every_phase_and_sanitized_evidence() { + let mut executor = FakeExecutor { + executions: (0..10).map(passed_execution).collect(), + calls: Vec::new(), + }; + let evidence = certify(&tiny_profile(), &mut executor).unwrap(); + assert_eq!(executor.calls, Phase::ALL); + assert_eq!(evidence.status, OutcomeStatus::Passed); + assert_eq!(evidence.phases.len(), 10); + assert!( + evidence + .phases + .iter() + .all(|phase| phase.peak_rss_bytes.is_some()) + ); + let encoded = serde_json::to_string(&evidence).unwrap(); + for forbidden in ["args", "stdout", "stderr", "path", "credential", "secret"] { + assert!(!encoded.contains(forbidden)); + } + } + + #[test] + fn first_failure_stops_before_later_public_commands() { + let mut executions: VecDeque<_> = (0..3).map(passed_execution).collect(); + executions.push_back(Ok(Execution { + exit_code: Some(7), + duration_ms: 4, + peak_rss_bytes: Some(4_096), + })); + executions.extend((4..10).map(passed_execution)); + let mut executor = FakeExecutor { + executions, + calls: Vec::new(), + }; + let mut events = Vec::new(); + let evidence = certify_with_events(&tiny_profile(), &mut executor, |event| { + events.push(event.clone()); + Ok(()) + }) + .unwrap(); + assert_eq!(evidence.failed_phase, Some(Phase::Reopen)); + assert_eq!(executor.calls, Phase::ALL[..4]); + assert_eq!(evidence.phases.len(), 4); + assert_eq!(events.len(), 4); + assert_eq!(events.last().unwrap().outcome.status, OutcomeStatus::Failed); + } + + #[test] + fn legacy_evidence_normalizes_to_current_typed_contract() { + let normalized = normalize_evidence( + br#"{"profile":"legacy-s20","phases":[{"name":"admission","ok":true,"duration_secs":1.25,"max_rss_kib":2,"exit_code":0}]}"#, + ) + .unwrap(); + assert_eq!(normalized.schema, EVIDENCE_SCHEMA); + assert_eq!(normalized.phases[0].duration_ms, 1_250); + assert_eq!(normalized.phases[0].peak_rss_bytes, Some(2_048)); + } + + #[test] + fn profile_rejects_non_graphforge_executable() { + let mut profile = tiny_profile(); + profile.executable = "/usr/bin/true".to_owned(); + assert_eq!( + profile.validate(), + Err(RunnerError::Profile( + "executable must resolve to the public gf command" + )) + ); + } + + #[test] + fn profile_rejects_noop_commands_labeled_as_lifecycle_phases() { + let mut profile = tiny_profile(); + for command in &mut profile.phases { + command.action = PhaseAction::GraphForgeCli { + args: vec!["--version".to_owned()], + }; + } + assert_eq!( + profile.validate(), + Err(RunnerError::Profile( + "phase action must select the matching benchmark or public gf operation" + )) + ); + } +} diff --git a/benchmarks/runners/certify/src/main.rs b/benchmarks/runners/certify/src/main.rs new file mode 100644 index 00000000..860431e0 --- /dev/null +++ b/benchmarks/runners/certify/src/main.rs @@ -0,0 +1,44 @@ +use graphforge_benchmark_certify::{ + PublicProcessExecutor, certify_with_events, normalize_evidence, read_profile, write_evidence, +}; +use std::env; +use std::fs; +use std::io::{self, Write}; +use std::path::Path; + +fn main() { + if let Err(message) = run() { + eprintln!("graphforge certification runner: {message}"); + std::process::exit(2); + } +} + +fn run() -> Result<(), &'static str> { + let arguments: Vec = env::args().skip(1).collect(); + match arguments.as_slice() { + [command, profile, evidence] if command == "run" => { + let profile = read_profile(Path::new(profile)).map_err(|_| "invalid profile")?; + let mut executor = PublicProcessExecutor; + let stdout = io::stdout(); + let mut output = stdout.lock(); + let outcome = certify_with_events(&profile, &mut executor, |event| { + serde_json::to_writer(&mut output, event) + .map_err(|_| graphforge_benchmark_certify::RunnerError::Io)?; + writeln!(output).map_err(|_| graphforge_benchmark_certify::RunnerError::Io) + }) + .map_err(|_| "certification failed")?; + output.flush().map_err(|_| "phase event write failed")?; + write_evidence(Path::new(evidence), &outcome).map_err(|_| "evidence write failed")?; + if outcome.failed_phase.is_some() { + std::process::exit(1); + } + Ok(()) + } + [command, input, output] if command == "normalize" => { + let input = fs::read(input).map_err(|_| "legacy evidence read failed")?; + let evidence = normalize_evidence(&input).map_err(|_| "legacy evidence invalid")?; + write_evidence(Path::new(output), &evidence).map_err(|_| "evidence write failed") + } + _ => Err("usage: graphforge-benchmark-certify "), + } +} diff --git a/benchmarks/schemas/certification-evidence.json b/benchmarks/schemas/certification-evidence.json new file mode 100644 index 00000000..8bc02dcc --- /dev/null +++ b/benchmarks/schemas/certification-evidence.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "schema": "graphforge-public-certification-evidence-schema/1", + "type": "object", + "properties": { + "schema": { "const": "graphforge-public-certification/1" }, + "profile_id": { "type": "string", "pattern": "^[A-Za-z0-9_.-]{1,80}$" }, + "status": { "enum": ["passed", "failed"] }, + "phases": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "items": { "$ref": "#/$defs/phase" } + }, + "failed_phase": { + "oneOf": [ + { "$ref": "#/$defs/phaseName" }, + { "type": "null" } + ] + } + }, + "required": ["schema", "profile_id", "status", "phases", "failed_phase"], + "additionalProperties": false, + "$defs": { + "phaseName": { + "enum": [ + "admission", + "generate", + "ingest", + "reopen", + "recount", + "query", + "export", + "verify", + "clean_import", + "reopen_proof" + ] + }, + "phase": { + "type": "object", + "properties": { + "phase": { "$ref": "#/$defs/phaseName" }, + "status": { "enum": ["passed", "failed"] }, + "duration_ms": { "type": "integer", "minimum": 0 }, + "peak_rss_bytes": { + "oneOf": [ + { "type": "integer", "minimum": 0 }, + { "type": "null" } + ] + }, + "exit_code": { + "oneOf": [ + { "type": "integer" }, + { "type": "null" } + ] + }, + "failure": { + "enum": ["command_failed", "command_unavailable"] + } + }, + "required": ["phase", "status", "duration_ms", "peak_rss_bytes", "exit_code"], + "allOf": [ + { + "if": { + "properties": { "status": { "const": "failed" } }, + "required": ["status"] + }, + "then": { "required": ["failure"] }, + "else": { "not": { "required": ["failure"] } } + } + ], + "additionalProperties": false + } + } +} diff --git a/benchmarks/schemas/certification-phase-event.json b/benchmarks/schemas/certification-phase-event.json new file mode 100644 index 00000000..0a1e9aff --- /dev/null +++ b/benchmarks/schemas/certification-phase-event.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "schema": "graphforge-public-certification-phase-event-schema/1", + "type": "object", + "properties": { + "schema": { "const": "graphforge-public-certification-phase-event/1" }, + "profile_id": { "type": "string", "pattern": "^[A-Za-z0-9_.-]{1,80}$" }, + "outcome": { "$ref": "certification-evidence.json#/$defs/phase" } + }, + "required": ["schema", "profile_id", "outcome"], + "additionalProperties": false +} diff --git a/benchmarks/schemas/certification-profile.json b/benchmarks/schemas/certification-profile.json new file mode 100644 index 00000000..946f49fc --- /dev/null +++ b/benchmarks/schemas/certification-profile.json @@ -0,0 +1,88 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "schema": "graphforge-public-certification-profile-schema/1", + "type": "object", + "properties": { + "schema": { "const": "graphforge-public-certification-profile/1" }, + "id": { "type": "string", "pattern": "^[A-Za-z0-9_.-]{1,80}$" }, + "executable": { "type": "string", "pattern": "(^|[/\\\\])gf(\\.exe)?$" }, + "phases": { + "type": "array", + "minItems": 10, + "maxItems": 10, + "prefixItems": [ + { "$ref": "#/$defs/admission" }, + { "$ref": "#/$defs/generate" }, + { "$ref": "#/$defs/ingest" }, + { "$ref": "#/$defs/reopen" }, + { "$ref": "#/$defs/recount" }, + { "$ref": "#/$defs/query" }, + { "$ref": "#/$defs/export" }, + { "$ref": "#/$defs/verify" }, + { "$ref": "#/$defs/cleanImport" }, + { "$ref": "#/$defs/reopenProof" } + ], + "items": false + } + }, + "required": ["schema", "id", "executable", "phases"], + "additionalProperties": false, + "$defs": { + "command": { + "type": "object", + "properties": { + "phase": { "type": "string" }, + "action": { + "oneOf": [ + { "$ref": "#/$defs/benchmarkGenerator" }, + { "$ref": "#/$defs/graphForgeCli" } + ] + } + }, + "required": ["phase", "action"], + "additionalProperties": false + }, + "benchmarkGenerator": { + "type": "object", + "properties": { + "interface": { "const": "benchmark_generator" }, + "identity": { "type": "string", "pattern": "^[A-Za-z0-9_.-]{1,80}$" }, + "executable": { "type": "string", "minLength": 1 }, + "args": { "type": "array", "items": { "type": "string" } } + }, + "required": ["interface", "identity", "executable", "args"], + "additionalProperties": false + }, + "graphForgeCli": { + "type": "object", + "properties": { + "interface": { "const": "graph_forge_cli" }, + "args": { "type": "array", "items": { "type": "string" } } + }, + "required": ["interface", "args"], + "additionalProperties": false + }, + "graphForgeCommand": { + "allOf": [ + { "$ref": "#/$defs/command" }, + { "properties": { "action": { "$ref": "#/$defs/graphForgeCli" } } } + ] + }, + "generatorCommand": { + "allOf": [ + { "$ref": "#/$defs/command" }, + { "properties": { "action": { "$ref": "#/$defs/benchmarkGenerator" } } } + ] + }, + "admission": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "admission" } } }] }, + "generate": { "allOf": [{ "$ref": "#/$defs/generatorCommand" }, { "properties": { "phase": { "const": "generate" } } }] }, + "ingest": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "ingest" } } }] }, + "reopen": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "reopen" } } }] }, + "recount": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "recount" } } }] }, + "query": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "query" } } }] }, + "export": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "export" } } }] }, + "verify": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "verify" } } }] }, + "cleanImport": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "clean_import" } } }] }, + "reopenProof": { "allOf": [{ "$ref": "#/$defs/graphForgeCommand" }, { "properties": { "phase": { "const": "reopen_proof" } } }] } + } +} diff --git a/benchmarks/tests/test_smoke.py b/benchmarks/tests/test_smoke.py index 27b4c8f5..48385f89 100644 --- a/benchmarks/tests/test_smoke.py +++ b/benchmarks/tests/test_smoke.py @@ -26,6 +26,30 @@ def test_smoke_evidence_schema_accepts_its_minimal_document(self) -> None: } ) + def test_public_certification_evidence_schema_accepts_sanitized_outcome(self) -> None: + schema = json.loads( + (workspace_root() / "schemas" / "certification-evidence.json").read_text( + encoding="utf-8" + ) + ) + Draft202012Validator(schema).validate( + { + "schema": "graphforge-public-certification/1", + "profile_id": "tiny-public-certification", + "status": "passed", + "phases": [ + { + "phase": "admission", + "status": "passed", + "duration_ms": 1, + "peak_rss_bytes": 1024, + "exit_code": 0, + } + ], + "failed_phase": None, + } + ) + def test_product_manifests_do_not_reference_benchmark_dependencies(self) -> None: repository = workspace_root().parent manifests = sorted(repository.rglob("Cargo.toml")) + sorted(