From 3c9934a12a134cf553e911fa074581d0f5573af5 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Fri, 24 Jul 2026 19:35:11 +0000 Subject: [PATCH 1/2] feat: run adversarial campaigns from recipes --- docs/recipe-contract.md | 32 +++ examples/recipes/adversarial-stateful.json | 100 +++++++ package.json | 2 +- packages/cli/src/adversarial-recipe.ts | 265 ++++++++++++++++++ packages/cli/src/cli-entry.ts | 3 + packages/cli/src/command-router.ts | 6 +- packages/cli/src/commands/adversarial.ts | 26 ++ .../cli/src/commands/recipe-run-finalizer.ts | 1 + packages/cli/src/commands/recipe-run-types.ts | 5 +- packages/cli/src/commands/recipe-run.ts | 27 +- packages/cli/src/output.ts | 2 + packages/cli/src/recipe-validation.ts | 68 ++++- .../runtime-core/src/adversarial-campaign.ts | 17 +- packages/runtime-core/src/recipe-schema.ts | 115 ++++++++ .../runtime-core/src/runtime-contracts.ts | 45 +++ tests/adversarial-campaign.test.ts | 25 ++ .../adversarial-recipe-orchestration.test.ts | 103 +++++++ tests/command-router.test.ts | 9 + 18 files changed, 835 insertions(+), 16 deletions(-) create mode 100644 examples/recipes/adversarial-stateful.json create mode 100644 packages/cli/src/adversarial-recipe.ts create mode 100644 packages/cli/src/commands/adversarial.ts create mode 100644 tests/adversarial-recipe-orchestration.test.ts diff --git a/docs/recipe-contract.md b/docs/recipe-contract.md index f9b9112f2..9f3902a4e 100644 --- a/docs/recipe-contract.md +++ b/docs/recipe-contract.md @@ -86,6 +86,7 @@ top-level fields: - `inputs` - `workflow` - `fuzzRun` +- `adversarialCampaigns` - `artifacts` - `probes` - `metadata` @@ -124,6 +125,37 @@ Use `inputs.workspace_preloads` for generic `agent-runtime/workspace-preload` artifact contracts. WP Codebox materializes declared repositories as sandbox workspace mounts; callers own the policy that decides which artifacts to pass. +## Adversarial Campaigns + +`adversarialCampaigns` additively declares deterministic corpus campaigns without +changing authored `fuzzRun.cases`. Each corpus action selects a registered +`caseTemplates` id. Generated plans interpolate `{{case.id}}`, `{{case.input}}`, +`{{action.input}}`, `{{action.inputBase64}}`, `{{matrix}}`, and +`{{matrix.}}` into template arguments, then execute the resulting phases +through the existing runtime-backed fuzz suite, checkpoint, episode, command +policy, and artifact lifecycle. + +The built-in mutator registry contains `scalar`, `structured`, `binary`, and +`sequence`; the initial neutral oracle registry contains `runtime-status`. +Required capabilities fail recipe validation before services or runtimes start. +Optional capabilities are reported with explicit availability in run evidence. +Concurrency above one requires an `isolated-workers` capability and therefore +fails closed until the selected recipe runtime provides isolated workers. + +Run and replay through stable CLI routes: + +```bash +wp-codebox adversarial run --recipe examples/recipes/adversarial-stateful.json --json +wp-codebox adversarial replay --recipe recipe.json --replay files/adversarial//replay/.json --json +``` + +Every campaign writes a bounded nested bundle under +`files/adversarial//`, including the result, fingerprint-addressed +findings, minimized replay envelopes, a manifest, and secret-scan evidence. The +parent artifact manifest retains all nested files. Interruption returns bounded +partial campaign evidence before the recipe runner performs its normal runtime +and service teardown. + ## External Service Boundaries Use `inputs.externalServices` to declare reviewer-safe boundaries for services a diff --git a/examples/recipes/adversarial-stateful.json b/examples/recipes/adversarial-stateful.json new file mode 100644 index 000000000..8155f5509 --- /dev/null +++ b/examples/recipes/adversarial-stateful.json @@ -0,0 +1,100 @@ +{ + "schema": "wp-codebox/workspace-recipe/v1", + "runtime": { + "backend": "wordpress" + }, + "workflow": { + "steps": [ + { + "command": "wordpress.run-php", + "args": ["code=delete_option( 'wp_codebox_adversarial_state' );"] + } + ] + }, + "adversarialCampaigns": [ + { + "schema": "wp-codebox/adversarial-recipe-campaign/v1", + "id": "neutral-stateful", + "seed": "neutral-stateful-v1", + "corpus": [ + { + "id": "option-seed", + "actions": [ + { + "type": "option-roundtrip", + "input": { + "value": "alpha" + } + } + ], + "input": { + "source": "recipe" + }, + "signals": ["baseline"] + } + ], + "caseTemplates": [ + { + "id": "option-roundtrip", + "phases": { + "action": [ + { + "command": "wordpress.run-php", + "args": ["code=$value = json_decode( base64_decode( '{{action.inputBase64}}' ), true ); update_option( 'wp_codebox_adversarial_state', $value );"] + } + ], + "assert": [ + { + "command": "wordpress.run-php", + "args": ["code=if ( get_option( 'wp_codebox_adversarial_state', null ) === null ) { throw new RuntimeException( 'state was not retained' ); }"] + } + ], + "teardown": [ + { + "command": "wordpress.run-php", + "args": ["code=delete_option( 'wp_codebox_adversarial_state' );"] + } + ] + } + } + ], + "mutators": ["scalar", "structured", "sequence"], + "oracles": [ + { + "id": "runtime-status", + "severity": "high" + } + ], + "matrix": [ + { + "name": "storage", + "values": ["options"] + } + ], + "concurrency": 1, + "budgets": { + "maxCases": 8, + "maxActionsPerCase": 8, + "maxInputBytes": 65536, + "maxCaseTimeMs": 10000, + "maxWallTimeMs": 120000, + "maxArtifactBytes": 10485760 + }, + "resetPolicy": { + "mode": "checkpoint-per-case", + "checkpointName": "neutral-stateful-baseline" + }, + "requiredCapabilities": ["adversarial-campaign", "artifact-export", "command:wordpress.run-php"], + "optionalCapabilities": ["transport-faults"], + "novelty": { + "retainSignals": true + }, + "shrinking": { + "enabled": true + }, + "replay": { + "selection": "findings" + } + } + ] +} diff --git a/package.json b/package.json index 558ce3a27..187fbd6db 100644 --- a/package.json +++ b/package.json @@ -230,7 +230,7 @@ "test:recipe-run-provenance": "tsx tests/recipe-run-provenance.test.ts", "test:fuzz-run-recipe": "tsx tests/fuzz-run-recipe.test.ts", "test:fuzz-suite-runner": "tsx tests/fuzz-suite-runner.test.ts", - "test:adversarial-runtime": "tsx --test tests/transport-faults.test.ts tests/adversarial-campaign.test.ts tests/adversarial-browser.test.ts tests/browser-clock-control.test.ts", + "test:adversarial-runtime": "tsx --test tests/transport-faults.test.ts tests/adversarial-campaign.test.ts tests/adversarial-browser.test.ts tests/browser-clock-control.test.ts tests/adversarial-recipe-orchestration.test.ts", "test:playground-fuzz-suite-public": "tsx tests/playground-fuzz-suite-public.test.ts", "test:nested-fuzz-suite-recipe-command": "tsx tests/nested-fuzz-suite-recipe-command.test.ts", "test:wordpress-fuzz-suite-builders": "tsx tests/wordpress-fuzz-suite-builders.test.ts", diff --git a/packages/cli/src/adversarial-recipe.ts b/packages/cli/src/adversarial-recipe.ts new file mode 100644 index 000000000..b27de44a2 --- /dev/null +++ b/packages/cli/src/adversarial-recipe.ts @@ -0,0 +1,265 @@ +import { createHash } from "node:crypto" +import { readFile, writeFile } from "node:fs/promises" +import { isAbsolute, join, relative } from "node:path" + +import { + ADVERSARIAL_ORACLE_SCHEMA, + adversarialCampaign, + artifactManifestFile, + refreshArtifactManifestFileSha256s, + runAdversarialCampaign, + upsertArtifactManifestFiles, + writeAdversarialEvidenceBundle, + type AdversarialCampaignResult, + type AdversarialCasePlan, + type AdversarialExecutionObservation, + type AdversarialReplay, + type ArtifactBundle, + type ArtifactManifest, + type Runtime, + type WorkspaceRecipe, + type WorkspaceRecipeAdversarialCampaign, + type WorkspaceRecipeFuzzCasePhase, + type WorkspaceRecipeStep, +} from "@automattic/wp-codebox-core" +import { stripUndefined } from "@automattic/wp-codebox-core/internals" + +import type { InputMountPathMapping } from "./input-mount-paths.js" +import { recipeAdversarialCapabilities } from "./recipe-validation.js" +import { executeRecipeWorkflowStep } from "./commands/recipe-run-workflow-evidence.js" +import type { RecipeExecutionResult, RecipeRunOptions } from "./commands/recipe-run-types.js" + +export interface RecipeAdversarialCampaignOutput { + declaration: WorkspaceRecipeAdversarialCampaign + result: AdversarialCampaignResult + capabilities: { + available: string[] + required: string[] + optional: Array<{ id: string; available: boolean }> + } + evidence?: Awaited> +} + +interface RunRecipeAdversarialCampaignsOptions { + recipe: WorkspaceRecipe + recipePath: string + recipeDirectory: string + runtime: Runtime + sandboxWorkspace?: Parameters[3] + artifactRoot?: string + runOptions?: RecipeRunOptions + inputMountPathMap?: readonly InputMountPathMapping[] + signal?: AbortSignal + executions: RecipeExecutionResult[] + provenance?: Record +} + +export async function runRecipeAdversarialCampaigns(options: RunRecipeAdversarialCampaignsOptions): Promise { + const outputs: RecipeAdversarialCampaignOutput[] = [] + const capabilities = recipeAdversarialCapabilities(options.recipe) + const replay = options.runOptions?.adversarialReplayPath ? await readAdversarialReplay(options.runOptions.adversarialReplayPath, options.recipeDirectory) : undefined + for (const declaration of options.recipe.adversarialCampaigns ?? []) { + if (replay && replay.campaignId !== declaration.id) continue + const templates = new Map(declaration.caseTemplates.map((template) => [template.id, template])) + const campaign = adversarialCampaign({ + id: declaration.id, + seed: declaration.seed, + corpus: declaration.corpus, + mutationKinds: declaration.mutators, + budgets: { ...declaration.budgets, workers: declaration.concurrency ?? 1 }, + oracles: declaration.oracles.map((oracle) => ({ ...oracle, schema: ADVERSARIAL_ORACLE_SCHEMA })), + matrix: declaration.matrix, + faults: declaration.faultSchedule, + provenance: stripUndefined({ + ...options.provenance, + recipe: recipePortableIdentity(options.recipe, options.recipePath), + components: options.recipe.inputs?.component_manifest, + mounts: recipeMountIdentities(options.recipe), + capabilities: { available: capabilities, required: declaration.requiredCapabilities ?? [], optional: declaration.optionalCapabilities ?? [] }, + resetPolicy: declaration.resetPolicy, + }), + }) + const execute = async (plan: AdversarialCasePlan, signal: AbortSignal) => executeRecipeAdversarialCase(declaration, templates, plan, signal, options) + const result = replay + ? await runRecipeAdversarialReplay(campaign, declaration, templates, replay, execute, options.signal) + : await runAdversarialCampaign(campaign, { + signal: options.signal, + retainNovelty: declaration.novelty?.retainSignals !== false, + minimize: declaration.shrinking?.enabled !== false, + replayCommand: (_campaign, _plan, fingerprint) => `wp-codebox adversarial replay --recipe ${portableRecipeRef(options.recipePath)} --replay files/adversarial/${declaration.id}/replay/${fingerprint}.json`, + execute, + }) + outputs.push({ + declaration, + result, + capabilities: { + available: capabilities, + required: declaration.requiredCapabilities ?? [], + optional: (declaration.optionalCapabilities ?? []).map((id) => ({ id, available: capabilities.includes(id) })), + }, + }) + } + return outputs +} + +async function runRecipeAdversarialReplay( + campaign: Parameters[0], + declaration: WorkspaceRecipeAdversarialCampaign, + templates: Map, + replay: AdversarialReplay, + execute: (plan: AdversarialCasePlan, signal: AbortSignal) => Promise, + signal?: AbortSignal, +): Promise { + const controller = new AbortController() + const abort = () => controller.abort(signal?.reason) + signal?.addEventListener("abort", abort, { once: true }) + try { + const plan: AdversarialCasePlan = { + id: replay.caseId, + caseId: replay.caseId, + corpusId: replay.corpusId, + iteration: replay.iteration, + workerId: replay.workerId, + matrix: replay.matrix, + actions: replay.actions, + input: replay.input, + mutation: { kind: declaration.mutators[0] ?? "sequence", path: "$", description: "recorded minimized replay" }, + } + for (const action of plan.actions) if (!templates.has(action.type)) throw new Error(`Replay references unknown case template ${action.type}.`) + const started = Date.now() + const observation = await execute(plan, controller.signal) + const signals = [...new Set(observation.signals ?? [])].sort() + return { + schema: "wp-codebox/adversarial-campaign-result/v1", + campaignId: campaign.id, + seed: replay.seed, + status: observation.status === "passed" ? "passed" : "findings", + summary: { generated: 1, executed: 1, retained: 1, findings: observation.status === "passed" ? 0 : 1, duplicates: 0, timedOut: observation.status === "timed-out" ? 1 : 0 }, + corpus: [{ id: replay.caseId, actions: replay.actions, input: replay.input, signals }], + findings: [], + schedule: replay.schedule, + noveltySignals: signals, + diagnostics: [{ code: "adversarial-replay-completed", message: `Replayed ${replay.caseId} through the recipe fuzz lifecycle.` }], + resourceUsage: { wallTimeMs: Date.now() - started, artifactBytes: (observation.artifacts ?? []).reduce((sum, artifact) => sum + (artifact.bytes ?? 0), 0) }, + } + } finally { + signal?.removeEventListener("abort", abort) + } +} + +async function readAdversarialReplay(path: string, recipeDirectory: string): Promise { + const resolved = isAbsolute(path) ? path : join(recipeDirectory, path) + const replay = JSON.parse(await readFile(resolved, "utf8")) as AdversarialReplay + if (replay.schema !== "wp-codebox/adversarial-replay/v1" || !replay.campaignId || !replay.caseId || !Array.isArray(replay.actions)) throw new Error(`Invalid adversarial replay envelope: ${path}`) + return replay +} + +async function executeRecipeAdversarialCase( + declaration: WorkspaceRecipeAdversarialCampaign, + templates: Map, + plan: AdversarialCasePlan, + signal: AbortSignal, + options: RunRecipeAdversarialCampaignsOptions, +): Promise { + if (signal.aborted) return { status: "error", diagnostics: [{ code: "campaign-case-interrupted", message: "Case was interrupted before execution." }] } + const phases = materializeCasePhases(plan, templates) + const suite = { + schema: "wp-codebox/fuzz-suite/v1", + id: `${declaration.id}-${plan.caseId}`, + target: { kind: "runtime", id: "wordpress.run-workload" }, + resetPolicy: declaration.resetPolicy ?? { mode: "none" }, + cases: [{ + id: plan.caseId, + input: { schema: "wp-codebox/wordpress-workload-run/v1", id: plan.caseId, steps: [], metadata: { adversarial: { campaignId: declaration.id, corpusId: plan.corpusId, iteration: plan.iteration, workerId: plan.workerId, matrix: plan.matrix, mutation: plan.mutation } } }, + phases, + metadata: { adversarialCase: true }, + }], + metadata: { adversarialCampaignId: declaration.id, faultSchedule: declaration.faultSchedule }, + } + const execution = await executeRecipeWorkflowStep(options.runtime, { + phase: "adversarial:action", + index: plan.iteration, + step: { command: "wp-codebox/run-fuzz-suite", args: [`input-json=${JSON.stringify(suite)}`], metadata: { campaignId: declaration.id, caseId: plan.caseId } }, + }, options.recipeDirectory, options.sandboxWorkspace, options.artifactRoot, options.runOptions, options.inputMountPathMap) + options.executions.push(execution) + const parsed = parseObject(execution.stdout) + const fuzzCase = Array.isArray(parsed?.cases) ? parseObjectValue(parsed.cases[0]) : undefined + const diagnostics = Array.isArray(fuzzCase?.diagnostics) ? fuzzCase.diagnostics.flatMap((item) => { + const diagnostic = parseObjectValue(item) + return diagnostic ? [{ code: String(diagnostic.code ?? "fuzz-suite-diagnostic"), message: String(diagnostic.message ?? "Fuzz suite diagnostic"), severity: typeof diagnostic.severity === "string" ? diagnostic.severity : undefined }] : [] + }) : [] + const artifactRefs = Array.isArray(fuzzCase?.artifactRefs) ? fuzzCase.artifactRefs.flatMap((item) => { + const ref = parseObjectValue(item) + return ref && typeof ref.path === "string" ? [{ path: ref.path, kind: String(ref.kind ?? "fuzz-artifact"), bytes: typeof ref.bytes === "number" ? ref.bytes : undefined, sha256: typeof ref.sha256 === "string" ? ref.sha256 : undefined }] : [] + }) : [] + const status = fuzzCase?.status === "passed" && execution.exitCode === 0 ? "passed" : fuzzCase?.status === "failed" ? "failed" : "error" + const signals = [ + `status:${status}`, + ...diagnostics.map((diagnostic) => `diagnostic:${diagnostic.code}`), + ...(typeof fuzzCase?.skipReason === "string" ? [`skip:${fuzzCase.skipReason}`] : []), + ] + return stripUndefined({ + status, + signals, + diagnostics, + artifacts: artifactRefs, + stateDigest: createHash("sha256").update(JSON.stringify({ status, signals, matrix: plan.matrix })).digest("hex"), + metadata: { fuzzSuite: parsed, resetPolicy: declaration.resetPolicy ?? { mode: "none" }, faultSchedule: declaration.faultSchedule }, + }) as AdversarialExecutionObservation +} + +function materializeCasePhases(plan: AdversarialCasePlan, templates: Map): Partial> { + const phases: Partial> = {} + for (const phase of ["setup", "action", "assert", "teardown"] as const) { + phases[phase] = plan.actions.flatMap((action) => (templates.get(action.type)?.phases[phase] ?? []).map((step) => materializeStep(step, plan, action.input))) + } + return phases +} + +function materializeStep(step: WorkspaceRecipeStep, plan: AdversarialCasePlan, actionInput: unknown): WorkspaceRecipeStep { + const replacements: Record = { + "{{case.id}}": plan.caseId, + "{{case.input}}": JSON.stringify(plan.input ?? null), + "{{action.input}}": JSON.stringify(actionInput ?? null), + "{{action.inputBase64}}": Buffer.from(JSON.stringify(actionInput ?? null)).toString("base64"), + "{{matrix}}": JSON.stringify(plan.matrix), + } + for (const [name, value] of Object.entries(plan.matrix)) replacements[`{{matrix.${name}}}`] = value + const replace = (value: string): string => Object.entries(replacements).reduce((result, [token, replacement]) => result.split(token).join(replacement), value) + return { ...step, args: step.args?.map(replace), metadata: { ...step.metadata, adversarialCaseId: plan.caseId, adversarialMutation: plan.mutation } } +} + +export async function writeRecipeAdversarialEvidence(artifacts: ArtifactBundle, campaigns: RecipeAdversarialCampaignOutput[], sensitiveValues: string[] = []): Promise { + if (campaigns.length === 0) return + const parentManifestPath = isAbsolute(artifacts.manifestPath) ? artifacts.manifestPath : join(artifacts.directory, artifacts.manifestPath) + const manifest = JSON.parse(await readFile(parentManifestPath, "utf8")) as ArtifactManifest + for (const campaign of campaigns) { + const directory = join(artifacts.directory, "files", "adversarial", campaign.declaration.id) + const evidence = await writeAdversarialEvidenceBundle(directory, campaign.result, { maxBytes: campaign.declaration.budgets?.maxArtifactBytes, sensitiveValues, createdAt: artifacts.createdAt }) + campaign.evidence = { ...evidence, path: relative(artifacts.directory, evidence.path) } + const paths = [evidence.manifestPath, evidence.resultPath, ...evidence.findingPaths, ...evidence.replayPaths, evidence.secretScanPath] + upsertArtifactManifestFiles(manifest, paths.map((path) => artifactManifestFile(relative(artifacts.directory, join(directory, path)), path === evidence.manifestPath ? "adversarial-evidence-manifest" : path === evidence.resultPath ? "adversarial-campaign-result" : path.startsWith("findings/") ? "adversarial-finding" : path.startsWith("replay/") ? "adversarial-replay" : "adversarial-secret-scan", "application/json"))) + } + await refreshArtifactManifestFileSha256s(artifacts.directory, manifest) + await writeFile(parentManifestPath, `${JSON.stringify(manifest, null, 2)}\n`) +} + +function recipePortableIdentity(recipe: WorkspaceRecipe, recipePath: string): Record { + return { schema: recipe.schema, ref: portableRecipeRef(recipePath), sha256: createHash("sha256").update(JSON.stringify(recipe)).digest("hex") } +} + +function recipeMountIdentities(recipe: WorkspaceRecipe): Array> { + return [...(recipe.runtime?.stack?.mounts ?? []), ...(recipe.inputs?.mounts ?? [])].map((mount) => ({ target: mount.target, mode: mount.mode ?? "readonly", type: mount.type ?? "directory", captureArtifacts: mount.captureArtifacts ?? true })) +} + +function portableRecipeRef(path: string): string { + return path.split(/[\\/]/).pop() || "recipe.json" +} + +function parseObject(value: string): Record | undefined { + try { return parseObjectValue(JSON.parse(value)) } catch { return undefined } +} + +function parseObjectValue(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined +} diff --git a/packages/cli/src/cli-entry.ts b/packages/cli/src/cli-entry.ts index b0035e255..ee8ebde9c 100644 --- a/packages/cli/src/cli-entry.ts +++ b/packages/cli/src/cli-entry.ts @@ -14,6 +14,7 @@ import { runFuzzDescriptorCommand, runFuzzMinimizeCaseCommand, runFuzzReadinessC import { runRunsArtifactsCommand, runRunsCancelCommand, runRunsStatusCommand } from "./commands/runs.js" import { runTargetProvisionCommand } from "./commands/target.js" import { runWorkspacePolicyCheckCommand } from "./commands/workspace-policy.js" +import { runAdversarialRecipeCommand, runAdversarialReplayCommand } from "./commands/adversarial.js" import { printHelp } from "./output.js" export async function runCli(args: string[]): Promise { @@ -29,6 +30,8 @@ export async function runCli(args: string[]): Promise { runWordPressWorkload: runWordPressWorkloadCommand, fuzzDescriptor: runFuzzDescriptorCommand, fuzzReadiness: runFuzzReadinessCommand, + adversarialRun: runAdversarialRecipeCommand, + adversarialReplay: runAdversarialReplayCommand, recipeValidate: runRecipeValidateCommand, recipeBuild: runRecipeBuildCommand, workspacePolicyCheck: runWorkspacePolicyCheckCommand, diff --git a/packages/cli/src/command-router.ts b/packages/cli/src/command-router.ts index 122833a4b..8bd8b61c9 100644 --- a/packages/cli/src/command-router.ts +++ b/packages/cli/src/command-router.ts @@ -33,6 +33,10 @@ const cliCommandRoutes = { readiness: "fuzzReadiness", capabilities: "fuzzReadiness", }, + adversarial: { + run: "adversarialRun", + replay: "adversarialReplay", + }, recipe: { validate: "recipeValidate", build: "recipeBuild", @@ -152,7 +156,7 @@ export async function routeCliCommand(argv: string[], router: CliCommandRouter): } async function maybeRespawnWithJspi(command: string | undefined, args: string[]): Promise { - if (!command || !["boot", "run", "recipe-run", "agent-task-run", "run-agent-task", "run-wordpress-workload"].includes(command)) { + if (!command || !["boot", "run", "recipe-run", "adversarial", "agent-task-run", "run-agent-task", "run-wordpress-workload"].includes(command)) { return undefined } diff --git a/packages/cli/src/commands/adversarial.ts b/packages/cli/src/commands/adversarial.ts new file mode 100644 index 000000000..cb6103c46 --- /dev/null +++ b/packages/cli/src/commands/adversarial.ts @@ -0,0 +1,26 @@ +import { runRecipeRunCommand } from "./recipe-run.js" + +export async function runAdversarialRecipeCommand(args: string[]): Promise { + return runRecipeRunCommand(args) +} + +export async function runAdversarialReplayCommand(args: string[]): Promise { + const mapped: string[] = [] + let hasReplay = false + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] as string + if (arg === "--replay") { + const value = args[++index] + if (!value) throw new Error("Missing value for --replay") + mapped.push("--adversarial-replay", value) + hasReplay = true + } else if (arg.startsWith("--replay=")) { + mapped.push(`--adversarial-replay=${arg.slice("--replay=".length)}`) + hasReplay = true + } else { + mapped.push(arg) + } + } + if (!hasReplay) throw new Error("Missing required option: --replay") + return runRecipeRunCommand(mapped) +} diff --git a/packages/cli/src/commands/recipe-run-finalizer.ts b/packages/cli/src/commands/recipe-run-finalizer.ts index 1a25b3a0c..b416f8617 100644 --- a/packages/cli/src/commands/recipe-run-finalizer.ts +++ b/packages/cli/src/commands/recipe-run-finalizer.ts @@ -364,6 +364,7 @@ function classifyRecipePhaseFailure(phase: string): string { case "import_fixture_databases": return "fixture_database" case "run_workloads": + case "run_adversarial_campaigns": return "workload" case "run_probes": return "probe" diff --git a/packages/cli/src/commands/recipe-run-types.ts b/packages/cli/src/commands/recipe-run-types.ts index a5b6e80d8..74a31dad0 100644 --- a/packages/cli/src/commands/recipe-run-types.ts +++ b/packages/cli/src/commands/recipe-run-types.ts @@ -4,6 +4,7 @@ import type { AgentSandboxResultSummary, AgentTaskSingleResult, RecipeReplayStat import type { RecipeExternalServiceBoundaryHostCorrelation } from "../recipe-external-services.js" import type { RecipeValidationIssue, RecipeWorkflowPhase } from "../recipe-validation.js" import type { RunOutput } from "../runtime-command-wrappers.js" +import type { RecipeAdversarialCampaignOutput } from "../adversarial-recipe.js" export interface RecipeRunOptions { recipePath: string @@ -20,6 +21,7 @@ export interface RecipeRunOptions { previewLeaseId?: string previewLeaseFile?: string timeoutMs: number + adversarialReplayPath?: string policy?: RuntimePolicy json: boolean summary: boolean @@ -78,6 +80,7 @@ export interface RecipeRunOutput { completionOutcome?: SandboxCompletionOutcome replayStatus?: RecipeReplayStatusSummary fuzzRun?: RecipeFuzzRunResult + adversarialCampaigns?: RecipeAdversarialCampaignOutput[] provenance?: RecipeRunProvenance result?: RecipeRunSummary artifacts?: ArtifactBundle @@ -276,7 +279,7 @@ export interface RecipeDiagnosticArtifactRef { sha256?: string } -export type RecipePhaseName = "provision_runtime_services" | "runtime_startup" | "mount_plugins" | "activate_plugins" | "run_blueprint_steps" | "apply_distribution" | "import_fixture_databases" | "run_distribution_setup_artifacts" | "run_distribution_startup_probes" | "run_workloads" | "run_probes" | "collect_artifacts" +export type RecipePhaseName = "provision_runtime_services" | "runtime_startup" | "mount_plugins" | "activate_plugins" | "run_blueprint_steps" | "apply_distribution" | "import_fixture_databases" | "run_distribution_setup_artifacts" | "run_distribution_startup_probes" | "run_workloads" | "run_adversarial_campaigns" | "run_probes" | "collect_artifacts" export interface RecipePhaseEvidence { schema: "wp-codebox/recipe-phase-evidence/v1" diff --git a/packages/cli/src/commands/recipe-run.ts b/packages/cli/src/commands/recipe-run.ts index c219072b1..c0fd6316b 100644 --- a/packages/cli/src/commands/recipe-run.ts +++ b/packages/cli/src/commands/recipe-run.ts @@ -31,6 +31,7 @@ import { importRecipeSiteSeeds } from "./recipe-site-seeds.js" import { applyRecipeRuntimeSetup, cleanupInputMountBaselines, prepareRecipeRuntimeSetup, recipeRunDependencyOverlay, recipeRunExtraPlugin, recipeRunStagedFile, rewriteInputMountPathArgs } from "./recipe-runtime-setup.js" import { provisionRuntimeServices, provisionRuntimeServicesForRecipe, runtimeServiceEvidenceFromError, type RuntimeServiceEvidence } from "../runtime-services.js" import { distributionStartupProbeFailure, executeRecipeCollectWorkloadResult, executeRecipeWorkflowStep, recipeAdvisoryFailure, recipeBrowserEvidence, recipeStepFailure, recipeWorkflowArgsEvidence, recipeWorkflowStepIsAdvisory, runDistributionSetupArtifacts, runDistributionStartupProbes, runRecipeProbes, withRecipeExecutionPhase } from "./recipe-run-workflow-evidence.js" +import { runRecipeAdversarialCampaigns, writeRecipeAdversarialEvidence, type RecipeAdversarialCampaignOutput } from "../adversarial-recipe.js" import type { RecipeAdvisoryFailure, RecipeBrowserEvidence, RecipeDiagnosticArtifactRef, RecipeEffectiveRecipeArtifact, RecipeExecutionResult, RecipeFuzzCaseCommandRef, RecipeFuzzCaseResult, RecipeFuzzCaseStatus, RecipeFuzzRunResult, RecipeInterruptionController, RecipePhaseEvidence, RecipePhaseName, RecipePhpWasmRuntimeDiagnostic, RecipeRunCommandOutput, RecipeRunComponentContract, RecipeRunDeclaredArtifact, RecipeRunDistributionSetupArtifact, RecipeRunDistributionStartupProbe, RecipeRunFixtureDatabase, RecipeRunOptions, RecipeRunOutput, RecipeRunProbe, RecipeRunProvenance, RecipeRunStagedFile, RecipeRuntimeDiagnostic, RecipeStepFailure, RecipeValidateOptions, RecipeValidateOutput } from "./recipe-run-types.js" const DEFAULT_RECIPE_RUN_TIMEOUT_MS = 25 * 60 * 1000 @@ -155,6 +156,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe let distributionStartupProbes: RecipeRunDistributionStartupProbe[] = [] let probes: RecipeRunProbe[] = [] let declaredArtifacts: RecipeRunDeclaredArtifact[] = [] + let adversarialCampaigns: RecipeAdversarialCampaignOutput[] = [] let stepFailures: RecipeStepFailure[] = [] let advisoryFailures: RecipeAdvisoryFailure[] = [] let browserEvidence: RecipeBrowserEvidence[] = [] @@ -315,6 +317,21 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe } }) + adversarialCampaigns = await phaseTracker.run("run_adversarial_campaigns", { campaigns: recipe.adversarialCampaigns?.map(({ id, seed }) => ({ id, seed })) ?? [] }, async () => runRecipeAdversarialCampaigns({ + recipe, + recipePath, + recipeDirectory, + runtime: runtime!, + sandboxWorkspace, + artifactRoot: configuredArtifactsDirectory, + runOptions: options, + inputMountPathMap, + signal: interruption?.signal, + executions, + provenance: recipeRunProvenance(recipe, recipePath) as unknown as Record, + })) + interruption?.throwIfInterrupted() + probes = await phaseTracker.run("run_probes", phaseProbeData(recipe), async () => await awaitRecipe("recipe-probes.run", runRecipeProbes(recipe, recipeDirectory, runtime!, executions))) const probeFailure = recipeProbeFailure(probes) if (probeFailure) { @@ -328,6 +345,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe await awaitRecipe("runtime.observe:mounts", runtime!.observe({ type: "mounts" })) runRecord = await runRegistry.update(runRecord.runId, { status: "collecting_artifacts", runtime: await runtime!.info() }) artifacts = await awaitRecipe("runtime.collect-artifacts", collectRecipeRuntimeArtifacts(runtime!, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS, activeExecution: executions.at(-1) })) + await writeRecipeAdversarialEvidence(artifacts, adversarialCampaigns, Object.values(secretEnv)) browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe) await artifactPointer.update({ runtime: await runtime!.info(), artifacts, phases: phaseTracker.list(), browserEvidence }) await materializeTypedRecipeDeclaredArtifacts(artifacts, declaredArtifacts) @@ -412,7 +430,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe browserEvidence, replayStatus: evidence.replayStatus ? recipeReplayStatusOutput(evidence.replayStatus) : undefined, failure: recipeFailure, - output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), provenance: recipeRunProvenance(recipe, recipePath) }, + output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), provenance: recipeRunProvenance(recipe, recipePath) }, }) } @@ -431,7 +449,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe phaseEvidence: phaseTracker.list(), browserEvidence, replayStatus: evidence.replayStatus ? recipeReplayStatusOutput(evidence.replayStatus) : undefined, - output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), provenance: recipeRunProvenance(recipe, recipePath) }, + output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), provenance: recipeRunProvenance(recipe, recipePath) }, }) } catch (error) { const failedServiceEvidence = runtimeServiceEvidenceFromError(error) @@ -477,6 +495,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe })) if (artifacts) { const collectedArtifacts = artifacts + await writeRecipeAdversarialEvidence(artifacts, adversarialCampaigns, Object.values(secretEnv)).catch(() => undefined) browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe) await artifactPointer.update({ runtime: await activeRuntime.info(), artifacts, phases: phaseTracker.list(), browserEvidence }) try { @@ -555,6 +574,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe ...(advisoryFailures.length > 0 ? { advisoryFailures } : {}), ...(browserEvidence.length > 0 ? { browserEvidence } : {}), ...(fuzzRunResult ? { fuzzRun: fuzzRunResult } : {}), + ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), diagnostics: recipeRuntimeDiagnostics(recipe, executions, error), provenance: recipeRunProvenance(recipe, recipePath, diagnosticArtifacts), }, @@ -799,6 +819,9 @@ function parseRecipeRunOptions(args: string[]): RecipeRunOptions { case "--policy": options.policy = parseRecipePolicy(value) break + case "--adversarial-replay": + options.adversarialReplayPath = value + break default: throw new Error(`Unknown option: ${name}`) } diff --git a/packages/cli/src/output.ts b/packages/cli/src/output.ts index 19384e680..61cb54179 100644 --- a/packages/cli/src/output.ts +++ b/packages/cli/src/output.ts @@ -328,6 +328,8 @@ export function printHelp(): void { wp-codebox preview-lease release (--registry --lease-id |--lease-file ) [--json] wp-codebox target provision [--id ] [--kind ] [--workspace-root ] [--json] wp-codebox run-fuzz-suite --input-file [--format=json] [--dry-run] [--runner-mode=simple|runtime-backed] + wp-codebox adversarial run --recipe [recipe-run options] + wp-codebox adversarial replay --recipe --replay [recipe-run options] wp-codebox fuzz-minimize-case --input-file [--format=json] [--dry-run] wp-codebox run-wordpress-workload --input-file [--format=json] [--dry-run] wp-codebox run-agent-task --input-file [--json] [--preview-hold-seconds ] [--preview-hold-blocking] [--preview-port ] [--preview-bind ] [--preview-public-url ] [--preview-lease-json ] diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index 8abe2f699..bff3cc6f7 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -1,6 +1,6 @@ import { readFile, stat } from "node:fs/promises" import { dirname, join, resolve } from "node:path" -import { assertFixtureImportDeterministicIdsSupported, assertWorkspaceRecipeJsonSchema, commandArgValue, normalizeRuntimeBackendKind, normalizeRuntimeMountTarget, parseCommandJson, safeArtifactRelativePath, validateBrowserInteractionScript, validateRuntimePolicy, validateSourcePackage, workspaceRecipeRuntimeCollectedArtifacts, type MountSpec, type RuntimeAssetSpec, type RuntimePolicy, type RuntimePreviewSpec, type WorkspaceRecipe, type WorkspaceRecipeDeclaredArtifact, type WorkspaceRecipeDependencyOverlay, type WorkspaceRecipeDistribution, type WorkspaceRecipeDistributionStartupProbe, type WorkspaceRecipeFixtureDatabase, type WorkspaceRecipeFuzzCasePhase, type WorkspaceRecipeMount, type WorkspaceRecipePluginRuntime, type WorkspaceRecipePluginRuntimeHealthProbe, type WorkspaceRecipeProbe, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeOverlay, type WorkspaceRecipeSiteSeed } from "@automattic/wp-codebox-core" +import { RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES, assertFixtureImportDeterministicIdsSupported, assertWorkspaceRecipeJsonSchema, commandArgValue, normalizeRuntimeBackendKind, normalizeRuntimeMountTarget, parseCommandJson, safeArtifactRelativePath, validateBrowserInteractionScript, validateRuntimePolicy, validateSourcePackage, workspaceRecipeRuntimeCollectedArtifacts, type MountSpec, type RuntimeAssetSpec, type RuntimePolicy, type RuntimePreviewSpec, type WorkspaceRecipe, type WorkspaceRecipeDeclaredArtifact, type WorkspaceRecipeDependencyOverlay, type WorkspaceRecipeDistribution, type WorkspaceRecipeDistributionStartupProbe, type WorkspaceRecipeFixtureDatabase, type WorkspaceRecipeFuzzCasePhase, type WorkspaceRecipeMount, type WorkspaceRecipePluginRuntime, type WorkspaceRecipePluginRuntimeHealthProbe, type WorkspaceRecipeProbe, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeOverlay, type WorkspaceRecipeSiteSeed } from "@automattic/wp-codebox-core" import { commandValidationDescriptorFor, effectivePolicyCommandsFor, type CommandArgValidationDescriptor } from "@automattic/wp-codebox-core/contracts" import { composerPackageVendorPath, evaluateRecipeSourcePolicy, isComposerPackageName, pluginTarget, recipeExtraPluginSlug, recipeExtraPluginSource, recipeExtraPluginSourceRoot, recipeExtraPluginSourceSubpath, recipeExtraPlugins, recipeSource, resolveRecipeExtraPluginFile } from "./recipe-sources.js" import { loadConfiguredRuntimeOverlayDescriptors, registeredRuntimeOverlayDescriptors, runtimeOverlayDescriptor, runtimeOverlayTarget } from "./runtime-overlay-registry.js" @@ -12,7 +12,7 @@ export interface RecipeValidationIssue { message: string } -export type RecipeWorkflowPhase = "setup" | "before" | "steps" | "after" | `fuzz:${WorkspaceRecipeFuzzCasePhase}` +export type RecipeWorkflowPhase = "setup" | "before" | "steps" | "after" | `fuzz:${WorkspaceRecipeFuzzCasePhase}` | `adversarial:${WorkspaceRecipeFuzzCasePhase}` export interface RecipeWorkflowStepRef { phase: Exclude @@ -225,6 +225,7 @@ export function validateWorkspaceRecipeShape(recipe: WorkspaceRecipe, recipePath validateFixtureDatabases(recipe.inputs?.fixtureDatabases, recipePath) validateRecipeProbes(recipe.probes, recipePath) validateRecipeFuzzRun(recipe.fuzzRun, recipePath) + validateRecipeAdversarialCampaigns(recipe, recipePath) validateDeclaredArtifacts(recipe.artifacts?.paths, recipePath) const siteSeeds = recipe.inputs?.siteSeeds ?? [] @@ -551,7 +552,7 @@ export async function validateWorkspaceRecipeSemantics(recipe: WorkspaceRecipe, await validateExistingBackendPackageSource(resolve(recipeDirectory, recipe.runtime.backendPackage.source), "$.runtime.backendPackage.source", addIssue) } - for (const { phase, index, step } of recipeWorkflowSteps(recipe)) { + for (const { phase, index, step } of recipeDeclaredWorkflowSteps(recipe)) { const path = `$.workflow.${phase}[${index}]` if (!supportedRecipeCommands.has(step.command) && !hostRecipeCommandPattern.test(step.command)) { addIssue("unsupported-command", `${path}.command`, `Unsupported recipe command: ${step.command}`) @@ -939,6 +940,19 @@ export function recipeWorkflowSteps(recipe: WorkspaceRecipe): RecipeWorkflowStep ] } +export function recipeDeclaredWorkflowSteps(recipe: WorkspaceRecipe): RecipeWorkflowStepRef[] { + return [...recipeWorkflowSteps(recipe), ...recipeAdversarialWorkflowSteps(recipe)] +} + +function recipeAdversarialWorkflowSteps(recipe: WorkspaceRecipe): RecipeWorkflowStepRef[] { + const phaseNames: WorkspaceRecipeFuzzCasePhase[] = ["setup", "action", "assert", "teardown"] + return (recipe.adversarialCampaigns ?? []).flatMap((campaign) => campaign.caseTemplates.flatMap((template) => phaseNames.flatMap((phase) => (template.phases[phase] ?? []).map((step, index) => ({ + phase: `adversarial:${phase}` as const, + index, + step, + }))))) +} + function recipeFuzzWorkflowSteps(recipe: WorkspaceRecipe): RecipeWorkflowStepRef[] { const phaseNames: WorkspaceRecipeFuzzCasePhase[] = ["setup", "action", "assert", "teardown"] return (recipe.fuzzRun?.cases ?? []).flatMap((fuzzCase, fuzzCaseIndex) => phaseNames.flatMap((fuzzPhase) => (fuzzCase.phases[fuzzPhase] ?? []).map((step, fuzzStepIndex) => ({ @@ -965,16 +979,16 @@ export function recipePolicy(recipe: WorkspaceRecipe): RuntimePolicy { return [] }) const commands = [ - ...effectivePolicyCommandsFor(recipeWorkflowSteps(recipe).map(({ step }) => step.command), cliRecipeCommandDefinitions), + ...effectivePolicyCommandsFor(recipeDeclaredWorkflowSteps(recipe).map(({ step }) => step.command), cliRecipeCommandDefinitions), ...effectivePolicyCommandsFor(boundedRuntimePlanCommands(recipe), cliRecipeCommandDefinitions), ...effectivePolicyCommandsFor(pluginRuntimeCommands, cliRecipeCommandDefinitions), ...effectivePolicyCommandsFor(distributionStartupProbeCommands, cliRecipeCommandDefinitions), ...effectivePolicyCommandsFor((recipe.probes ?? []).map((probe) => probe.step.command), cliRecipeCommandDefinitions), ] - if (recipeWorkflowSteps(recipe).some(({ step }) => step.command === "wordpress.bench")) { + if (recipeDeclaredWorkflowSteps(recipe).some(({ step }) => step.command === "wordpress.bench")) { commands.unshift("wordpress.run-php") } - if (recipeWorkflowSteps(recipe).some(({ step }) => step.command === "wordpress.bench" && recipeBenchStepUsesWpCli(step))) { + if (recipeDeclaredWorkflowSteps(recipe).some(({ step }) => step.command === "wordpress.bench" && recipeBenchStepUsesWpCli(step))) { commands.unshift("wordpress.wp-cli") } if (recipeExtraPlugins(recipe).length > 0) { @@ -992,7 +1006,7 @@ export function recipePolicy(recipe: WorkspaceRecipe): RuntimePolicy { // Auto-grant the evaluate capability when a browser-actions step opts into the // arbitrary-JS escape hatch by including an evaluate step. Recipe authors opt in // by writing the step; direct `run` invocations still control the gate via --policy. - if (recipeWorkflowSteps(recipe).some(({ step }) => (step.command === "wordpress.browser-actions" || step.command === "wordpress.browser-scenario") && recipeStepUsesEvaluate(step))) { + if (recipeDeclaredWorkflowSteps(recipe).some(({ step }) => (step.command === "wordpress.browser-actions" || step.command === "wordpress.browser-scenario") && recipeStepUsesEvaluate(step))) { commands.push("wordpress.browser-actions.evaluate") } @@ -1122,6 +1136,46 @@ function validateRecipeFuzzRun(fuzzRun: WorkspaceRecipe["fuzzRun"] | undefined, } } +export function recipeAdversarialCapabilities(recipe: WorkspaceRecipe): string[] { + return [...new Set([ + "adversarial-campaign", + "artifact-export", + "bounded-evidence", + "replay", + ...RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES.capabilities, + ...listCliRecipeCommandIds().map((command) => `command:${command}`), + ])] +} + +function validateRecipeAdversarialCampaigns(recipe: WorkspaceRecipe, recipePath: string): void { + const campaigns = recipe.adversarialCampaigns + if (campaigns === undefined) return + if (!Array.isArray(campaigns) || campaigns.length === 0) throw new Error(`Recipe adversarialCampaigns must be a non-empty array: ${recipePath}`) + const campaignIds = new Set() + const available = new Set(recipeAdversarialCapabilities(recipe)) + for (const [campaignIndex, campaign] of campaigns.entries()) { + const campaignPath = `$.adversarialCampaigns[${campaignIndex}]` + if (!campaign || typeof campaign !== "object" || Array.isArray(campaign)) throw new Error(`Recipe adversarial campaigns must be objects at ${campaignPath}: ${recipePath}`) + if (!campaign.id || typeof campaign.id !== "string") throw new Error(`Recipe adversarial campaign requires id at ${campaignPath}: ${recipePath}`) + if (!Array.isArray(campaign.caseTemplates) || campaign.caseTemplates.length === 0) throw new Error(`Recipe adversarial campaign requires caseTemplates at ${campaignPath}: ${recipePath}`) + if (!Array.isArray(campaign.corpus) || campaign.corpus.length === 0) throw new Error(`Recipe adversarial campaign requires corpus at ${campaignPath}: ${recipePath}`) + if (campaignIds.has(campaign.id)) throw new Error(`Recipe adversarial campaign ids must be unique: ${campaign.id}: ${recipePath}`) + campaignIds.add(campaign.id) + const templateIds = new Set(campaign.caseTemplates.map((template) => template.id)) + if (templateIds.size !== campaign.caseTemplates.length) throw new Error(`Recipe adversarial case template ids must be unique at ${campaignPath}: ${recipePath}`) + for (const entry of campaign.corpus) { + for (const action of entry.actions) { + if (!templateIds.has(action.type)) throw new Error(`Recipe adversarial corpus action references unknown case template ${action.type} at ${campaignPath}: ${recipePath}`) + } + } + const missing = (campaign.requiredCapabilities ?? []).filter((capability) => !available.has(capability)) + if (missing.length > 0) throw new Error(`Recipe adversarial campaign ${campaign.id} requires unavailable capabilities: ${missing.join(", ")}: ${recipePath}`) + if ((campaign.concurrency ?? 1) > 1 && !(campaign.requiredCapabilities ?? []).includes("isolated-workers")) { + throw new Error(`Recipe adversarial campaign ${campaign.id} concurrency above 1 requires isolated-workers capability: ${recipePath}`) + } + } +} + function validateDeclaredArtifacts(paths: WorkspaceRecipeDeclaredArtifact[] | undefined, recipePath: string): void { if (paths && !Array.isArray(paths)) { throw new Error(`Recipe artifacts.paths must be an array: ${recipePath}`) diff --git a/packages/runtime-core/src/adversarial-campaign.ts b/packages/runtime-core/src/adversarial-campaign.ts index 6b356bddf..a989f4620 100644 --- a/packages/runtime-core/src/adversarial-campaign.ts +++ b/packages/runtime-core/src/adversarial-campaign.ts @@ -152,7 +152,10 @@ export interface AdversarialCampaignRunnerOptions { execute(plan: AdversarialCasePlan, signal: AbortSignal): Promise evaluate?(plan: AdversarialCasePlan, observation: AdversarialExecutionObservation, oracles: readonly AdversarialOracleContract[]): Promise | AdversarialOracleResult[] now?: () => number - replayCommand?: (campaign: AdversarialCampaign, plan: AdversarialCasePlan) => string + replayCommand?: (campaign: AdversarialCampaign, plan: AdversarialCasePlan, fingerprint: string) => string + signal?: AbortSignal + retainNovelty?: boolean + minimize?: boolean } export interface DifferentialCell { @@ -209,6 +212,7 @@ export async function runAdversarialCampaign(campaignInput: AdversarialCampaign, let incomplete = false for (let round = 0; generated < campaign.budgets.maxCases; round += 1) { + if (options.signal?.aborted) { incomplete = true; diagnostics.push({ code: "campaign-interrupted", message: "Campaign stopped after an interruption request." }); break } if ((options.now ?? Date.now)() - started >= campaign.budgets.maxWallTimeMs) { incomplete = true; diagnostics.push({ code: "campaign-wall-time-exhausted", message: "Campaign stopped at its wall-time budget." }); break } const roundPlans: AdversarialCasePlan[] = [] for (let workerId = 0; workerId < campaign.budgets.workers && generated < campaign.budgets.maxCases; workerId += 1) { @@ -229,12 +233,13 @@ export async function runAdversarialCampaign(campaignInput: AdversarialCampaign, const newSignals = (observation.signals ?? []).filter((signal) => !novelty.has(signal)) for (const signal of newSignals) novelty.add(signal) - if (newSignals.length > 0) corpus.push({ id: plan.caseId, actions: plan.actions, input: plan.input, signals: [...new Set(observation.signals ?? [])], metadata: { retainedFrom: plan.corpusId, mutation: plan.mutation } }) + if (newSignals.length > 0 && options.retainNovelty !== false) corpus.push({ id: plan.caseId, actions: plan.actions, input: plan.input, signals: [...new Set(observation.signals ?? [])], metadata: { retainedFrom: plan.corpusId, mutation: plan.mutation } }) const oracleResults = options.evaluate ? await options.evaluate(plan, observation, campaign.oracles) : defaultOracleResults(observation) const failedOracles = oracleResults.filter((result) => result.failed) if (failedOracles.length > 0 || observation.status !== "passed") { - const minimized = await minimizeAdversarialCase(campaign, plan, failedOracles, options) + if (options.signal?.aborted) { incomplete = true; diagnostics.push({ code: "campaign-interrupted", message: "Campaign stopped before finding minimization after an interruption request." }); break } + const minimized = options.minimize === false ? normalizeCorpusEntry({ id: plan.caseId, actions: plan.actions, input: plan.input }) : await minimizeAdversarialCase(campaign, plan, failedOracles, options) const fingerprint = adversarialFindingFingerprint({ oracleIds: failedOracles.map((result) => result.oracleId), status: observation.status, diagnosticCodes: (observation.diagnostics ?? []).map((item) => item.code).sort(), matrix: plan.matrix }) const existing = findings.get(fingerprint) if (existing) existing.duplicates += 1 @@ -290,6 +295,8 @@ export function classifyDifferentialResult(cells: DifferentialCell[]): Different async function executeBoundedCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, options: AdversarialCampaignRunnerOptions): Promise { const controller = new AbortController() + const abort = () => controller.abort(options.signal?.reason) + options.signal?.addEventListener("abort", abort, { once: true }) let timer: NodeJS.Timeout | undefined try { return await Promise.race([ @@ -300,6 +307,7 @@ async function executeBoundedCase(campaign: AdversarialCampaign, plan: Adversari return { status: "error", diagnostics: [{ code: "case-execution-error", message: error instanceof Error ? error.message : String(error) }] } } finally { if (timer) clearTimeout(timer) + options.signal?.removeEventListener("abort", abort) } } @@ -330,6 +338,7 @@ function mutateCorpusEntry(campaign: AdversarialCampaign, source: AdversarialCor async function minimizeAdversarialCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, originalOracles: AdversarialOracleResult[], options: AdversarialCampaignRunnerOptions): Promise { const oracleIds = new Set(originalOracles.filter((item) => item.failed).map((item) => item.oracleId)) const preserves = async (candidate: AdversarialCorpusEntry): Promise => { + if (options.signal?.aborted) return false const candidatePlan = { ...plan, actions: candidate.actions, input: candidate.input } const observation = await executeBoundedCase(campaign, candidatePlan, options) const oracleResults = options.evaluate ? await options.evaluate(candidatePlan, observation, campaign.oracles) : defaultOracleResults(observation) @@ -354,7 +363,7 @@ async function minimizeAdversarialCase(campaign: AdversarialCampaign, plan: Adve } function createFinding(campaign: AdversarialCampaign, plan: AdversarialCasePlan, minimized: AdversarialCorpusEntry, observation: AdversarialExecutionObservation, oracles: AdversarialOracleResult[], schedule: AdversarialScheduleEntry[], options: AdversarialCampaignRunnerOptions, fingerprint: string): AdversarialFinding { - const command = options.replayCommand?.(campaign, plan) ?? campaign.replayCommand ?? `wp-codebox adversarial replay --campaign ${campaign.id} --case ${plan.caseId}` + const command = options.replayCommand?.(campaign, plan, fingerprint) ?? campaign.replayCommand ?? `wp-codebox adversarial replay --campaign ${campaign.id} --case ${plan.caseId}` return { schema: ADVERSARIAL_FINDING_SCHEMA, fingerprint, diff --git a/packages/runtime-core/src/recipe-schema.ts b/packages/runtime-core/src/recipe-schema.ts index 4ab69f271..89458f486 100644 --- a/packages/runtime-core/src/recipe-schema.ts +++ b/packages/runtime-core/src/recipe-schema.ts @@ -252,6 +252,12 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche }, }, fuzzRun: { $ref: "#/$defs/fuzzRun" }, + adversarialCampaigns: { + type: "array", + minItems: 1, + maxItems: 32, + items: { $ref: "#/$defs/adversarialRecipeCampaign" }, + }, artifacts: { type: "object", additionalProperties: false, @@ -1243,6 +1249,115 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche }, }, }, + adversarialRecipeCampaign: { + type: "object", + additionalProperties: false, + required: ["schema", "id", "seed", "corpus", "caseTemplates", "mutators", "oracles"], + properties: { + schema: { const: "wp-codebox/adversarial-recipe-campaign/v1" }, + id: { type: "string", pattern: "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, + seed: { type: "string", minLength: 1 }, + corpus: { type: "array", minItems: 1, maxItems: 1000, items: { $ref: "#/$defs/adversarialCorpusEntry" } }, + caseTemplates: { type: "array", minItems: 1, maxItems: 100, items: { $ref: "#/$defs/adversarialCaseTemplate" } }, + mutators: { type: "array", minItems: 1, uniqueItems: true, items: { enum: ["scalar", "structured", "binary", "sequence"] } }, + oracles: { type: "array", minItems: 1, maxItems: 32, items: { $ref: "#/$defs/adversarialOracle" } }, + matrix: { type: "array", maxItems: 16, items: { $ref: "#/$defs/adversarialMatrixDimension" } }, + concurrency: { type: "integer", minimum: 1, maximum: 128 }, + budgets: { $ref: "#/$defs/adversarialBudgets" }, + faultSchedule: { $ref: "#/$defs/transportFaultModel" }, + resetPolicy: { $ref: "#/$defs/adversarialResetPolicy" }, + requiredCapabilities: { type: "array", uniqueItems: true, items: { type: "string", minLength: 1 } }, + optionalCapabilities: { type: "array", uniqueItems: true, items: { type: "string", minLength: 1 } }, + novelty: { type: "object", additionalProperties: false, properties: { retainSignals: { type: "boolean" } } }, + shrinking: { type: "object", additionalProperties: false, properties: { enabled: { type: "boolean" } } }, + replay: { type: "object", additionalProperties: false, properties: { selection: { enum: ["findings", "all"] } } }, + metadata: { $ref: "#/$defs/metadata" }, + }, + }, + adversarialCorpusEntry: { + type: "object", + additionalProperties: false, + required: ["id", "actions"], + properties: { + id: { type: "string", pattern: "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, + actions: { type: "array", minItems: 1, maxItems: 10000, items: { $ref: "#/$defs/adversarialAction" } }, + input: {}, + signals: { type: "array", uniqueItems: true, items: { type: "string" } }, + metadata: { $ref: "#/$defs/metadata" }, + }, + }, + adversarialAction: { + type: "object", + additionalProperties: false, + required: ["type"], + properties: { type: { type: "string", pattern: "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, input: {}, metadata: { $ref: "#/$defs/metadata" } }, + }, + adversarialCaseTemplate: { + type: "object", + additionalProperties: false, + required: ["id", "phases"], + properties: { + id: { type: "string", pattern: "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, + phases: { + type: "object", + additionalProperties: false, + properties: { + setup: { type: "array", items: { $ref: "#/$defs/step" } }, + action: { type: "array", items: { $ref: "#/$defs/step" } }, + assert: { type: "array", items: { $ref: "#/$defs/step" } }, + teardown: { type: "array", items: { $ref: "#/$defs/step" } }, + }, + }, + }, + }, + adversarialOracle: { + type: "object", + additionalProperties: false, + required: ["id", "severity"], + properties: { id: { const: "runtime-status" }, severity: { enum: ["low", "medium", "high", "critical"] }, description: { type: "string" }, metadata: { $ref: "#/$defs/metadata" } }, + }, + adversarialMatrixDimension: { + type: "object", + additionalProperties: false, + required: ["name", "values"], + properties: { name: { type: "string", pattern: "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, values: { type: "array", minItems: 1, maxItems: 100, uniqueItems: true, items: { type: "string" } } }, + }, + adversarialBudgets: { + type: "object", + additionalProperties: false, + properties: { + maxCases: { type: "integer", minimum: 1, maximum: 100000 }, + maxActionsPerCase: { type: "integer", minimum: 1, maximum: 10000 }, + maxInputBytes: { type: "integer", minimum: 1, maximum: 67108864 }, + maxCaseTimeMs: { type: "integer", minimum: 1, maximum: 3600000 }, + maxWallTimeMs: { type: "integer", minimum: 1, maximum: 86400000 }, + maxArtifactBytes: { type: "integer", minimum: 1, maximum: 4294967296 }, + }, + }, + adversarialResetPolicy: { + type: "object", + additionalProperties: false, + required: ["mode"], + properties: { + mode: { enum: ["none", "checkpoint-per-case", "restore-snapshot"] }, + checkpointName: { type: "string" }, + snapshotRef: { type: "string" }, + fixtureRefs: { type: "array", items: { type: "string" } }, + metadata: { $ref: "#/$defs/metadata" }, + }, + }, + transportFaultModel: { + type: "object", + additionalProperties: false, + required: ["schema", "seed", "rules"], + properties: { + schema: { const: "wp-codebox/transport-fault-model/v1" }, + seed: { type: "string", minLength: 1 }, + rules: { type: "array", items: { type: "object" } }, + redactHeaders: { type: "array", items: { type: "string" } }, + metadata: { $ref: "#/$defs/metadata" }, + }, + }, inheritanceRequest: { type: "object", additionalProperties: false, diff --git a/packages/runtime-core/src/runtime-contracts.ts b/packages/runtime-core/src/runtime-contracts.ts index 9ea0d0c7e..10c1ec783 100644 --- a/packages/runtime-core/src/runtime-contracts.ts +++ b/packages/runtime-core/src/runtime-contracts.ts @@ -320,6 +320,50 @@ export interface WorkspaceRecipeFuzzRun { metadata?: Record } +export interface WorkspaceRecipeAdversarialCaseTemplate { + id: string + phases: Partial> +} + +export interface WorkspaceRecipeAdversarialCampaign { + schema: "wp-codebox/adversarial-recipe-campaign/v1" + id: string + seed: string + corpus: Array<{ + id: string + actions: Array<{ type: string; input?: unknown; metadata?: Record }> + input?: unknown + signals?: string[] + metadata?: Record + }> + caseTemplates: WorkspaceRecipeAdversarialCaseTemplate[] + mutators: Array<"scalar" | "structured" | "binary" | "sequence"> + oracles: Array<{ + id: "runtime-status" + severity: "low" | "medium" | "high" | "critical" + description?: string + metadata?: Record + }> + matrix?: Array<{ name: string; values: string[] }> + concurrency?: number + budgets?: Partial<{ + maxCases: number + maxActionsPerCase: number + maxInputBytes: number + maxCaseTimeMs: number + maxWallTimeMs: number + maxArtifactBytes: number + }> + faultSchedule?: import("./transport-faults.js").TransportFaultModel + resetPolicy?: import("./fuzz-suite-contracts.js").FuzzSuiteResetPolicy + requiredCapabilities?: string[] + optionalCapabilities?: string[] + novelty?: { retainSignals?: boolean } + shrinking?: { enabled?: boolean } + replay?: { selection?: "findings" | "all" } + metadata?: Record +} + export interface WorkspaceRecipeFixtureDatabaseReset { strategy?: "none" | "truncate-tables" tables?: string[] @@ -631,6 +675,7 @@ export interface WorkspaceRecipe { after?: WorkspaceRecipeStep[] } fuzzRun?: WorkspaceRecipeFuzzRun + adversarialCampaigns?: WorkspaceRecipeAdversarialCampaign[] artifacts?: { directory?: string verify?: boolean | WorkspaceRecipeArtifactVerifier diff --git a/tests/adversarial-campaign.test.ts b/tests/adversarial-campaign.test.ts index 15045c7e5..5eb5b4c6e 100644 --- a/tests/adversarial-campaign.test.ts +++ b/tests/adversarial-campaign.test.ts @@ -97,3 +97,28 @@ test("sealed finding bundles redact secrets and machine-specific paths", async ( await rm(directory, { recursive: true, force: true }) } }) + +test("campaign interruption stops scheduling and returns bounded partial evidence", async () => { + const controller = new AbortController() + let active = 0 + const result = await runAdversarialCampaign(adversarialCampaign({ + id: "interrupted", + seed: "interrupted-seed", + corpus: [{ id: "seed", actions: [{ type: "observe" }] }], + budgets: { maxCases: 100, workers: 2, maxCaseTimeMs: 1000, maxWallTimeMs: 10000, maxArtifactBytes: 1024 }, + }), { + signal: controller.signal, + execute: async (_plan, signal) => { + active += 1 + controller.abort("test interruption") + await new Promise((resolve) => setTimeout(resolve, 5)) + active -= 1 + return { status: signal.aborted ? "error" : "passed", artifacts: [{ path: "partial.json", kind: "partial", bytes: 16 }] } + }, + }) + assert.equal(result.status, "incomplete") + assert.equal(active, 0, "all active workers settle before interruption returns") + assert(result.summary.generated <= 2, "no later round is scheduled after interruption") + assert(result.resourceUsage.artifactBytes <= 1024) + assert(result.diagnostics.some((diagnostic) => diagnostic.code === "campaign-interrupted")) +}) diff --git a/tests/adversarial-recipe-orchestration.test.ts b/tests/adversarial-recipe-orchestration.test.ts new file mode 100644 index 000000000..8d809079a --- /dev/null +++ b/tests/adversarial-recipe-orchestration.test.ts @@ -0,0 +1,103 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" + +import { assertWorkspaceRecipeJsonSchema, type ArtifactBundle, type ExecutionSpec, type Runtime, type WorkspaceRecipe } from "../packages/runtime-core/src/index.js" +import { runRecipeAdversarialCampaigns, writeRecipeAdversarialEvidence } from "../packages/cli/src/adversarial-recipe.js" +import { recipePolicy, validateWorkspaceRecipeSemantics, validateWorkspaceRecipeShape } from "../packages/cli/src/recipe-validation.js" + +const recipe: WorkspaceRecipe = { + schema: "wp-codebox/workspace-recipe/v1", + workflow: { steps: [{ command: "inspect-mounted-inputs" }] }, + adversarialCampaigns: [{ + schema: "wp-codebox/adversarial-recipe-campaign/v1", + id: "neutral-state", + seed: "deterministic-seed", + corpus: [{ id: "seed", actions: [{ type: "option-roundtrip", input: { value: "alpha" } }], input: { state: 1 }, signals: ["seed"] }], + caseTemplates: [{ + id: "option-roundtrip", + phases: { + action: [{ command: "wordpress.run-php", args: ["code=echo '{{action.input}}';"] }], + assert: [{ command: "wordpress.run-php", args: ["code=echo '{{case.id}}';"] }], + }, + }], + mutators: ["scalar"], + oracles: [{ id: "runtime-status", severity: "high" }], + matrix: [{ name: "runtime", values: ["neutral"] }], + concurrency: 1, + budgets: { maxCases: 2, maxCaseTimeMs: 5000, maxWallTimeMs: 10000, maxArtifactBytes: 100000 }, + resetPolicy: { mode: "checkpoint-per-case", checkpointName: "baseline" }, + requiredCapabilities: ["adversarial-campaign", "artifact-export", "command:wordpress.run-php"], + optionalCapabilities: ["transport-faults"], + }], +} + +assertWorkspaceRecipeJsonSchema(recipe, { recipeCommandIds: ["inspect-mounted-inputs", "wordpress.run-php"] }) +validateWorkspaceRecipeShape(recipe, "recipe.json") +assert.deepEqual(await validateWorkspaceRecipeSemantics(recipe, "recipe.json"), []) +assert(recipePolicy(recipe).commands.includes("wordpress.run-php"), "template commands must participate in policy derivation") + +const executions: ExecutionSpec[] = [] +const checkpointOperations: string[] = [] +const runtime = { + info: async () => ({ id: "neutral", backend: "neutral", environment: { kind: "wordpress", name: "Neutral" }, createdAt: "2026-01-01T00:00:00.000Z", status: "created" }), + execute: async (spec: ExecutionSpec) => { + executions.push(spec) + return { id: `execution-${executions.length}`, command: spec.command, args: spec.args ?? [], exitCode: 0, stdout: "ok\n", stderr: "", startedAt: "2026-01-01T00:00:00.000Z", finishedAt: "2026-01-01T00:00:00.001Z" } + }, + createCheckpoint: async ({ name }: { name: string }) => { + checkpointOperations.push(`create:${name}`) + return { schema: "wp-codebox/runtime-checkpoint-result/v1", operation: "create", status: "created", name, supported: true } + }, + restoreCheckpoint: async (name: string) => { + checkpointOperations.push(`restore:${name}`) + return { schema: "wp-codebox/runtime-checkpoint-result/v1", operation: "restore", status: "restored", name, supported: true } + }, +} as unknown as Runtime + +const executeCampaign = async () => runRecipeAdversarialCampaigns({ + recipe, + recipePath: "/portable/recipe.json", + recipeDirectory: "/portable", + runtime, + executions: [], + provenance: { runtime: "neutral" }, +}) + +const first = await executeCampaign() +const second = await executeCampaign() +assert.equal(first[0]?.result.status, "passed") +assert.deepEqual(first[0]?.result.corpus, second[0]?.result.corpus) +assert.deepEqual(first[0]?.result.schedule, second[0]?.result.schedule) +assert.deepEqual(first[0]?.result.findings, second[0]?.result.findings) +assert.equal(first[0]?.capabilities.optional[0]?.available, false, "optional fidelity must be explicit") +assert(executions.length > 0, "generated cases must execute through runtime commands") +assert(checkpointOperations.includes("create:baseline") && checkpointOperations.includes("restore:baseline"), "campaign cases must use the existing checkpoint reset path") + +const unsupportedRecipe = structuredClone(recipe) +unsupportedRecipe.adversarialCampaigns![0]!.requiredCapabilities = ["missing-adapter"] +assert.throws(() => validateWorkspaceRecipeShape(unsupportedRecipe, "unsupported.json"), /requires unavailable capabilities: missing-adapter/) + +const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-adversarial-recipe-")) +try { + const manifestPath = join(artifactRoot, "manifest.json") + await writeFile(manifestPath, `${JSON.stringify({ id: "parent", createdAt: "2026-01-01T00:00:00.000Z", runtime: await runtime.info(), files: [] }, null, 2)}\n`) + const artifacts = { directory: artifactRoot, manifestPath, createdAt: "2026-01-01T00:00:00.000Z" } as unknown as ArtifactBundle + await writeRecipeAdversarialEvidence(artifacts, first) + const parentManifest = JSON.parse(await readFile(manifestPath, "utf8")) as { files: Array<{ path: string }> } + assert(parentManifest.files.some((file) => file.path === "files/adversarial/neutral-state/manifest.json")) + assert(parentManifest.files.some((file) => file.path.endsWith("adversarial-campaign-result.json"))) + assert.equal(first[0]?.evidence?.path, "files/adversarial/neutral-state") +} finally { + await rm(artifactRoot, { recursive: true, force: true }) +} + +const staticRecipe: WorkspaceRecipe = { + schema: "wp-codebox/workspace-recipe/v1", + workflow: { steps: [{ command: "inspect-mounted-inputs" }] }, + fuzzRun: { schema: "wp-codebox/fuzz-run/v1", cases: [{ case_id: "static", phases: { action: [{ command: "wordpress.run-php" }] } }] }, +} +assert.equal(staticRecipe.fuzzRun.cases.length, 1, "static fuzzRun declarations remain unchanged") + +console.log("adversarial recipe orchestration ok") diff --git a/tests/command-router.test.ts b/tests/command-router.test.ts index 5b570504f..e7929f03d 100644 --- a/tests/command-router.test.ts +++ b/tests/command-router.test.ts @@ -21,6 +21,11 @@ const router = { return 5 }, fuzzReadiness: async () => 0, + adversarialRun: async () => 0, + adversarialReplay: async (args: string[]) => { + calls.push(`adversarialReplay:${args.join(" ")}`) + return 9 + }, recipeValidate: async () => 0, recipeBuild: async () => 0, workspacePolicyCheck: async () => 0, @@ -69,4 +74,8 @@ const fuzzDescriptorExitCode = await routeCliCommand(["fuzz", "descriptor", "--j assert.equal(fuzzDescriptorExitCode, 5) assert.deepEqual(calls, ["agentTaskRun:--input-file task.json --json", "runtimeDescriptor:--json", "fuzzDescriptor:--json"]) +const adversarialReplayExitCode = await routeCliCommand(["adversarial", "replay", "--recipe", "recipe.json", "--replay", "replay.json", "--json"], router) +assert.equal(adversarialReplayExitCode, 9) +assert.equal(calls.at(-1), "adversarialReplay:--recipe recipe.json --replay replay.json --json") + console.log("command-router contract passed") From 94f9f260e8489dd1b3b88fc46b44850ff0c0b6ce Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Fri, 24 Jul 2026 19:35:53 +0000 Subject: [PATCH 2/2] fix: fail closed on unapplied fault schedules --- packages/cli/src/recipe-validation.ts | 3 +++ tests/adversarial-recipe-orchestration.test.ts | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index bff3cc6f7..d3fd54b37 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -1170,6 +1170,9 @@ function validateRecipeAdversarialCampaigns(recipe: WorkspaceRecipe, recipePath: } const missing = (campaign.requiredCapabilities ?? []).filter((capability) => !available.has(capability)) if (missing.length > 0) throw new Error(`Recipe adversarial campaign ${campaign.id} requires unavailable capabilities: ${missing.join(", ")}: ${recipePath}`) + if (campaign.faultSchedule && !(campaign.requiredCapabilities ?? []).includes("transport-faults")) { + throw new Error(`Recipe adversarial campaign ${campaign.id} faultSchedule requires the transport-faults capability: ${recipePath}`) + } if ((campaign.concurrency ?? 1) > 1 && !(campaign.requiredCapabilities ?? []).includes("isolated-workers")) { throw new Error(`Recipe adversarial campaign ${campaign.id} concurrency above 1 requires isolated-workers capability: ${recipePath}`) } diff --git a/tests/adversarial-recipe-orchestration.test.ts b/tests/adversarial-recipe-orchestration.test.ts index 8d809079a..b0ba3ddf8 100644 --- a/tests/adversarial-recipe-orchestration.test.ts +++ b/tests/adversarial-recipe-orchestration.test.ts @@ -79,6 +79,10 @@ const unsupportedRecipe = structuredClone(recipe) unsupportedRecipe.adversarialCampaigns![0]!.requiredCapabilities = ["missing-adapter"] assert.throws(() => validateWorkspaceRecipeShape(unsupportedRecipe, "unsupported.json"), /requires unavailable capabilities: missing-adapter/) +const unsafeFaultRecipe = structuredClone(recipe) +unsafeFaultRecipe.adversarialCampaigns![0]!.faultSchedule = { schema: "wp-codebox/transport-fault-model/v1", seed: "faults", rules: [] } +assert.throws(() => validateWorkspaceRecipeShape(unsafeFaultRecipe, "faults.json"), /faultSchedule requires the transport-faults capability/) + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-adversarial-recipe-")) try { const manifestPath = join(artifactRoot, "manifest.json")