From 4f5385e110b914c38ee4da7277c88b62bfe374af Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Sat, 19 Sep 2026 16:47:03 +0300 Subject: [PATCH 1/4] fix(toolchain): one registry decides which binaries the platform names The CLI display and the probe registry each wrote out the same nine tools. They had drifted: the display said mas-sentry, which is the string find_mst hands to shutil.which, while the probe registry said mst and wrote that name into every run manifest. regression_gate reported toolchain drift against a binary no resolver would ever look for. _TOOLCHAIN is now derived from TOOL_PROBES, so the two guards that already stood over the display -- that it names every finder in the package, and that each label is the binary its finder looks for -- now reach the manifest as well. Measured: restoring the mst spelling kills test_every_displayed_name_is_the_binary_its_finder_looks_for; dropping a probe kills test_the_status_display_names_every_finder_that_exists. Neither guard could see the probe registry before. Importing the probe registry into the CLI costs 0.0011 s and one module: the entry point already imported all nine finders directly. --- cyberai/__main__.py | 30 ++++++++++------------------ cyberai/bench/environment.py | 2 +- tests/unit/test_bench_environment.py | 2 +- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/cyberai/__main__.py b/cyberai/__main__.py index 01c2f4b2..88b2a42b 100644 --- a/cyberai/__main__.py +++ b/cyberai/__main__.py @@ -10,15 +10,7 @@ from cyberai.version import __version__ -from .agents.exploit.nuclei_engine import find_nuclei -from .agents.exploit.searchsploit import find_searchsploit -from .agents.mcp_scan.mst_bridge import find_mst -from .agents.recon.nmap_tool import find_nmap -from .agents.web3.aderyn_tool import find_aderyn -from .agents.web3.anvil_harness import find_anvil -from .agents.web3.foundry_poc import find_forge -from .agents.web3.halmos_tool import find_halmos -from .agents.web3.slither_tool import find_slither +from .bench.environment import TOOL_PROBES from .cli.bench import bench from .cli.detector_eval import detector from .cli.mcp_scan import mcp_scan @@ -33,17 +25,15 @@ # binary name -> the resolver that already locates it for the agents. Not a # second lookup: status calls the same functions the tools call, so a name # reported here is the one that will be executed. -_TOOLCHAIN = { - "nmap": find_nmap, - "nuclei": find_nuclei, - "searchsploit": find_searchsploit, - "forge": find_forge, - "aderyn": find_aderyn, - "slither": find_slither, - "anvil": find_anvil, - "halmos": find_halmos, - "mas-sentry": find_mst, -} +# +# Derived from the probe registry rather than written out a second time. The +# two copies had drifted: this one said "mas-sentry", which is the string the +# finder hands to shutil.which, while the probe registry said "mst" and wrote +# that name into every run manifest. A toolchain drift was reported against a +# binary no resolver would ever find. With one registry the two guards over +# this display -- that it names every finder in the package, and that each +# label is the binary its finder looks for -- reach the manifest as well. +_TOOLCHAIN = {probe.name: probe.resolver for probe in TOOL_PROBES} def _detach_stdin_from_tty() -> None: diff --git a/cyberai/bench/environment.py b/cyberai/bench/environment.py index 3f88386d..cdf53824 100644 --- a/cyberai/bench/environment.py +++ b/cyberai/bench/environment.py @@ -82,7 +82,7 @@ class ToolProbe: ToolProbe("forge", find_forge, "--version"), ToolProbe("anvil", find_anvil, "--version"), ToolProbe("halmos", find_halmos, "--version"), - ToolProbe("mst", find_mst, None), + ToolProbe("mas-sentry", find_mst, None), ) diff --git a/tests/unit/test_bench_environment.py b/tests/unit/test_bench_environment.py index f4bd3cb1..232381b8 100644 --- a/tests/unit/test_bench_environment.py +++ b/tests/unit/test_bench_environment.py @@ -102,7 +102,7 @@ def test_every_probe_resolves_through_the_module_that_owns_the_tool(): "forge": ("cyberai.agents.web3.foundry_poc", "find_forge"), "anvil": ("cyberai.agents.web3.anvil_harness", "find_anvil"), "halmos": ("cyberai.agents.web3.halmos_tool", "find_halmos"), - "mst": ("cyberai.agents.mcp_scan.mst_bridge", "find_mst"), + "mas-sentry": ("cyberai.agents.mcp_scan.mst_bridge", "find_mst"), } got = {p.name: (p.resolver.__module__, p.resolver.__name__) for p in env.TOOL_PROBES} assert got == expected From f37c3cf2eeb631ed8196b1f1dab282502662df98 Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Sat, 19 Sep 2026 17:03:19 +0300 Subject: [PATCH 2/4] feat(status): the screen can name the version of every tool it found status named nine binaries and said nothing about which build of them would run, while environment.py has measured exactly that since day 48. The two halves existed and never met. --versions runs each located tool's version flag and prints the number beside the name, or the reason there is none: searchsploit and mas-sentry carry flag=None, so they report 'version flag not measured' rather than a guess. Off by default, on the same split the bench CLI makes -- it imports the probe only when a manifest is asked for, because a probe starts a process per tool. status is the command an operator runs when something is already wrong, and nine hung binaries at the manifest's budget would hold that answer for three minutes. Measured: a binary that never answers costs exactly the timeout, 20.02 s at VERSION_TIMEOUT, 3.0 s at STATUS_VERSION_TIMEOUT. Both halves of the display come out of one probe pass. A second walk over the resolvers can disagree with the first, which is the defect the previous commit removed between the CLI and the run manifest. Mutation: four mutants, four killed. Two of them survived the first version of the tests. The cheap path was asserted with no tool installed, so 'no process started' held whatever the code did -- the shape of GU, a zero measured on a population where the signal cannot appear. The one-pass claim was asserted by counting probe calls, which a second walk over the resolvers never touches. Both tests now make the binary resolvable first. --- README.md | 2 +- blog/launch-post-draft.md | 2 +- cyberai/__main__.py | 42 ++++++++++++-- cyberai/bench/environment.py | 24 ++++++-- tests/unit/test_status_versions.py | 91 ++++++++++++++++++++++++++++++ 5 files changed, 148 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_status_versions.py diff --git a/README.md b/README.md index b9b9b104..58038580 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-2946%20collected-brightgreen) +![Tests](https://img.shields.io/badge/tests-2951%20collected-brightgreen) ![Mypy](https://img.shields.io/badge/mypy-strict%3A%20100%2F172%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index 0bf8cf60..189e7291 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,7 +13,7 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -2946 tests collected under the gated selection run before every commit, with the +2951 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` clean over 100 of 172 modules, Apache-2.0. diff --git a/cyberai/__main__.py b/cyberai/__main__.py index 88b2a42b..b3765fc0 100644 --- a/cyberai/__main__.py +++ b/cyberai/__main__.py @@ -10,7 +10,7 @@ from cyberai.version import __version__ -from .bench.environment import TOOL_PROBES +from .bench.environment import STATUS_VERSION_TIMEOUT, TOOL_PROBES, probe_toolchain from .cli.bench import bench from .cli.detector_eval import detector from .cli.mcp_scan import mcp_scan @@ -512,15 +512,40 @@ def _seed_line(llm: LLMConfig) -> str: return str(llm.seed) -def _toolchain_lines() -> tuple[str, str]: +def _toolchain_lines(*, versions: bool = False) -> tuple[str, str]: """Which external binaries resolve right now, both sides named. Both halves are sets of names rather than counts: "7 of 8 found" would say nothing about which one is missing, and a run that needs the missing one fails later with no hint that status already knew. + + ``versions`` runs each located binary's version flag. Off by default + because this resolves paths and starts no process, while a probe spends + up to STATUS_VERSION_TIMEOUT on every tool that answers slowly -- and + status is the command an operator reaches for when something is already + wrong. The same split the bench CLI makes, which imports the probe only + when a manifest is asked for. + + Under ``versions`` both halves come out of one probe pass rather than a + second walk over the resolvers: two walks can disagree, which is the + shape of defect that had the display and the manifest naming the same + binary differently. """ - found = [name for name, finder in _TOOLCHAIN.items() if finder() is not None] - missing = [name for name in _TOOLCHAIN if name not in found] + if not versions: + found = [name for name, finder in _TOOLCHAIN.items() if finder() is not None] + missing = [name for name in _TOOLCHAIN if name not in found] + return ( + ", ".join(found) if found else "none", + ", ".join(missing) if missing else "none", + ) + + probed = probe_toolchain(timeout=STATUS_VERSION_TIMEOUT) + found = [ + f"{t.name} {t.version}" if t.version else f"{t.name} ({t.detail})" + for t in probed + if t.path is not None + ] + missing = [t.name for t in probed if t.path is None] return ( ", ".join(found) if found else "none", ", ".join(missing) if missing else "none", @@ -542,7 +567,12 @@ def _api_key_line(llm: LLMConfig) -> str: @cli.command() -def status() -> None: +@click.option( + "--versions", + is_flag=True, + help="Run each located tool's version flag (starts processes)", +) +def status(versions: bool) -> None: """Show CyberAI status and config.""" config = CyberAIConfig.from_env() # The guard a real client would build from this config, not a second @@ -550,7 +580,7 @@ def status() -> None: # value; printing the raw setting would name a policy that never acts. guard = LLMClient(config.llm).guard classifier = guard.classifier - tools_found, tools_missing = _toolchain_lines() + tools_found, tools_missing = _toolchain_lines(versions=versions) console.print( Panel( f"Provider: {config.llm.provider}\n" diff --git a/cyberai/bench/environment.py b/cyberai/bench/environment.py index cdf53824..b114688b 100644 --- a/cyberai/bench/environment.py +++ b/cyberai/bench/environment.py @@ -53,6 +53,14 @@ VERSION_TIMEOUT = 20 +# Shorter than VERSION_TIMEOUT, and the difference is the caller, not the tool. +# A benchmark manifest is written once and has to record a version even from a +# tool that starts slowly. `status` is the command an operator runs when +# something is already wrong, and nine hung binaries at VERSION_TIMEOUT would +# hold that answer for three minutes. Measured: a binary that never answers +# costs exactly the timeout, 20.02 s at the default. +STATUS_VERSION_TIMEOUT = 3 + NOT_INSTALLED = "not installed" FLAG_NOT_MEASURED = "version flag not measured" NOT_PARSED = "version not parsed" @@ -107,15 +115,19 @@ def parse_version(stdout: str, stderr: str) -> str | None: return None -def probe_tool(probe: ToolProbe) -> ToolVersion: - """Resolve one binary and read its version. Never raises.""" +def probe_tool(probe: ToolProbe, *, timeout: float = VERSION_TIMEOUT) -> ToolVersion: + """Resolve one binary and read its version. Never raises. + + ``timeout`` is keyword-only and defaults to the manifest's budget, so the + callers that were here before this parameter existed read the same way. + """ path = probe.resolver() if path is None: return ToolVersion(probe.name, None, None, NOT_INSTALLED) if probe.flag is None: return ToolVersion(probe.name, path, None, FLAG_NOT_MEASURED) try: - proc = run_sealed([path, probe.flag], timeout=VERSION_TIMEOUT) + proc = run_sealed([path, probe.flag], timeout=timeout) except (subprocess.SubprocessError, OSError) as exc: return ToolVersion(probe.name, path, None, f"probe failed: {type(exc).__name__}") version = parse_version(proc.stdout, proc.stderr) @@ -124,11 +136,13 @@ def probe_tool(probe: ToolProbe) -> ToolVersion: return ToolVersion(probe.name, path, version, "") -def probe_toolchain(probes: Sequence[ToolProbe] = TOOL_PROBES) -> tuple[ToolVersion, ...]: +def probe_toolchain( + probes: Sequence[ToolProbe] = TOOL_PROBES, *, timeout: float = VERSION_TIMEOUT +) -> tuple[ToolVersion, ...]: """Probe every binary the platform drives, in registry order. A tuple, not a list: the result is stored on a frozen RunManifest and fingerprinted, and a mutable sequence there would be a shared reference into a record that claims to be immutable. """ - return tuple(probe_tool(probe) for probe in probes) + return tuple(probe_tool(probe, timeout=timeout) for probe in probes) diff --git a/tests/unit/test_status_versions.py b/tests/unit/test_status_versions.py new file mode 100644 index 00000000..59add0a9 --- /dev/null +++ b/tests/unit/test_status_versions.py @@ -0,0 +1,91 @@ +"""`status --versions` asks the tools, and plain `status` asks nothing. + +Two halves of one contract. The flag exists because a probe starts a process +per located tool and status is the command an operator runs when something is +already wrong; a default that pays that cost would be the defect the flag was +added to avoid. So the cheap path has to stay provably cheap: the assertion is +that no process starts, not that the output looks the same. +""" + +from __future__ import annotations + +from click.testing import CliRunner + +from cyberai import __main__ as main +from cyberai.bench import environment as env +from cyberai.bench.run_manifest import ToolVersion + + +def test_plain_status_starts_no_process(tmp_path, monkeypatch) -> None: + """A tool has to be found before "nothing ran" means anything. + + The first version of this test patched run_sealed on a machine with no + tools installed: the probe returned "not installed" before reaching any + process, so the assertion held whatever the code did. Mutating the flag + to probe unconditionally left it green. The stub below makes the binary + resolvable, so a probe would reach run_sealed and the assertion has + something to fail on. + """ + stub = tmp_path / "nmap" + stub.write_text("#!/bin/sh\necho 'Nmap version 7.98 ( https://nmap.org )'\n") + stub.chmod(0o755) + monkeypatch.setenv("NMAP_PATH", str(stub)) + + def boom(*args: object, **kwargs: object) -> object: + raise AssertionError("status ran a binary without --versions") + + monkeypatch.setattr(env, "run_sealed", boom) + result = CliRunner().invoke(main.cli, ["status"]) + assert result.exit_code == 0 + assert "nmap" in result.output + + +def test_the_flag_reports_the_version_the_tool_prints(tmp_path, monkeypatch) -> None: + stub = tmp_path / "nmap" + stub.write_text("#!/bin/sh\necho 'Nmap version 7.98 ( https://nmap.org )'\n") + stub.chmod(0o755) + monkeypatch.setenv("NMAP_PATH", str(stub)) + result = CliRunner().invoke(main.cli, ["status", "--versions"]) + assert "nmap 7.98" in result.output + + +def test_a_located_tool_with_no_flag_says_so_rather_than_guessing(tmp_path, monkeypatch) -> None: + """searchsploit and mas-sentry carry flag=None: no version was ever measured.""" + stub = tmp_path / "searchsploit" + stub.write_text("#!/bin/sh\necho 'Usage: searchsploit'\n") + stub.chmod(0o755) + monkeypatch.setenv("SEARCHSPLOIT_PATH", str(stub)) + result = CliRunner().invoke(main.cli, ["status", "--versions"]) + assert f"searchsploit ({env.FLAG_NOT_MEASURED})" in result.output + + +def test_the_probe_budget_is_shorter_than_the_manifest_budget() -> None: + """Three minutes of hung binaries is not an answer to "what is wrong".""" + assert env.STATUS_VERSION_TIMEOUT < env.VERSION_TIMEOUT + + +def test_one_pass_decides_both_halves(monkeypatch) -> None: + """Found and missing come from the same probe result, not two walks. + + Two walks over the resolvers can disagree, and a display that disagrees + with itself is the defect this file's sibling commit removed between the + CLI and the run manifest. + """ + resolved: list[str] = [] + probed: list[str] = [] + + def probe(probes=env.TOOL_PROBES, **kwargs: object): # type: ignore[no-untyped-def] + probed.append("once") + return tuple(ToolVersion(p.name, "/somewhere/" + p.name, "1.0", "") for p in probes) + + monkeypatch.setattr(main, "probe_toolchain", probe) + monkeypatch.setattr( + main, + "_TOOLCHAIN", + {name: (lambda n=name: resolved.append(n) or None) for name in dict(main._TOOLCHAIN)}, + ) + result = CliRunner().invoke(main.cli, ["status", "--versions"]) + + assert probed == ["once"] + assert resolved == [], f"a second walk asked the resolvers again: {resolved}" + assert "Tools missing: none" in result.output From fe09e4e7ffdfdbf64b0a1af2aa36171d9e0b0b24 Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Sat, 19 Sep 2026 17:17:18 +0300 Subject: [PATCH 3/4] feat(status): the screen shows what governs the run The panel printed twelve lines and none of them was strict_scope -- the one control that ends a run before the first phase since day 50. Six more that the orchestrator reads were equally invisible: the cost budget, the planner, replan, model routing, web recon, planned redteam. An operator could read the whole screen and still not know what the next scan would do. The rule is scanned, not listed: a test reads orchestrator.py and requires a line for every config field it consumes. It fails in both directions, so a control added to the orchestrator without a line fails, and a label left behind by a control the orchestrator stopped reading fails too. Written this way on a measurement: four fields on CyberAIConfig -- intel, timeout, verbose, use_lab_dogfood -- are read by nobody, so a rule saying every field would have demanded a line for levers that move nothing. use_lab_dogfood has three tests asserting the value reaches the object and no consumer at all. Strict scope is the only line that names its source, because it is the only control here that defaults to on: off has to answer who turned it off, while off-by-default against six flags that are already off answers nothing. The source is read as set-or-not rather than inferred from the value, since every reader in config.py returns the default for unset and for unparseable alike. Mutation: four mutants, four killed, including one that removes the orchestrator's read of a field to prove the rule fails from that side too. --- README.md | 4 +- blog/launch-post-draft.md | 2 +- cyberai/__main__.py | 42 +++++++++- ...est_the_readme_names_what_status_prints.py | 9 ++ tests/unit/test_status_versions.py | 82 +++++++++++++++++++ 5 files changed, 135 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 58038580..65ac536b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-2951%20collected-brightgreen) +![Tests](https://img.shields.io/badge/tests-2955%20collected-brightgreen) ![Mypy](https://img.shields.io/badge/mypy-strict%3A%20100%2F172%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) @@ -76,7 +76,7 @@ cyberai scan example.com --dry-run # real scan with a local model (air-gapped, no cloud) and scope cyberai scan app.target.com --provider ollama --scope "*.target.com" -cyberai status # provider, output, trust boundary, sampling, air-gap, credentials, toolchain +cyberai status # provider, output, trust boundary, sampling, air-gap, credentials, toolchain, run controls cyberai replay # re-run a saved session ``` diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index 189e7291..6ba4752a 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,7 +13,7 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -2951 tests collected under the gated selection run before every commit, with the +2955 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` clean over 100 of 172 modules, Apache-2.0. diff --git a/cyberai/__main__.py b/cyberai/__main__.py index b3765fc0..ff257284 100644 --- a/cyberai/__main__.py +++ b/cyberai/__main__.py @@ -552,6 +552,39 @@ def _toolchain_lines(*, versions: bool = False) -> tuple[str, str]: ) +def _budget_line(config: CyberAIConfig) -> str: + """The LLM spend ceiling. Zero disables the check rather than forbidding + spend, so it is spelled out: a panel that printed "0.0" would read as a + budget of nothing.""" + if config.max_cost_usd <= 0: + return "disabled" + return f"{config.max_cost_usd} USD" + + +def _strict_scope_line(config: CyberAIConfig) -> str: + """Whether the run refuses an unauthorised target, and who decided. + + The only line on this panel that names its source. strict_scope is the + one control here that defaults to on, and the one whose value ends a run + before the first phase, so "off" has to answer who turned it off. The + other controls default to off, and "(default)" against six of them would + be noise rather than an answer. + + The source is read from the environment rather than inferred from the + value: every reader in config.py returns the default for an unset + variable and for an unparseable one alike, so the value cannot say which + happened. What is stated here is what can be checked -- the variable is + set, or it is not. + """ + state = "on" if config.strict_scope else "off" + # `is None`, not truthiness: an empty variable is set, and reporting it as + # the default would be the panel answering about a value when the question + # is about the environment. + set_here = os.getenv("CYBERAI_STRICT_SCOPE") is not None + source = "CYBERAI_STRICT_SCOPE" if set_here else "default" + return f"{state} ({source})" + + def _api_key_line(llm: LLMConfig) -> str: """Whether the credential this provider needs is present. @@ -594,7 +627,14 @@ def status(versions: bool) -> None: f"Air-gapped: {'on' if config.air_gapped else 'off'}\n" f"API key: {_api_key_line(config.llm)}\n" f"Tools found: {tools_found}\n" - f"Tools missing: {tools_missing}", + f"Tools missing: {tools_missing}\n" + f"Strict scope: {_strict_scope_line(config)}\n" + f"Cost budget: {_budget_line(config)}\n" + f"Planner: {'on' if config.enable_planner else 'off'}\n" + f"Replan: {'on' if config.enable_replan else 'off'}\n" + f"Model routing: {'on' if config.routing.enable_model_routing else 'off'}\n" + f"Web recon: {'on' if config.use_web_recon else 'off'}\n" + f"Planned redteam: {'on' if config.use_planned_redteam else 'off'}", title="CyberAI Status", ) ) diff --git a/tests/architecture/test_the_readme_names_what_status_prints.py b/tests/architecture/test_the_readme_names_what_status_prints.py index e8c04c9b..ed97b11d 100644 --- a/tests/architecture/test_the_readme_names_what_status_prints.py +++ b/tests/architecture/test_the_readme_names_what_status_prints.py @@ -26,6 +26,15 @@ "air-gap": ["Air-gapped"], "credentials": ["API key"], "toolchain": ["Tools found", "Tools missing"], + "run controls": [ + "Strict scope", + "Cost budget", + "Planner", + "Replan", + "Model routing", + "Web recon", + "Planned redteam", + ], } # Both files are located from this file rather than from the imported diff --git a/tests/unit/test_status_versions.py b/tests/unit/test_status_versions.py index 59add0a9..cb6d0a23 100644 --- a/tests/unit/test_status_versions.py +++ b/tests/unit/test_status_versions.py @@ -9,6 +9,9 @@ from __future__ import annotations +import ast +import pathlib + from click.testing import CliRunner from cyberai import __main__ as main @@ -59,6 +62,85 @@ def test_a_located_tool_with_no_flag_says_so_rather_than_guessing(tmp_path, monk assert f"searchsploit ({env.FLAG_NOT_MEASURED})" in result.output +def test_the_screen_names_the_source_of_the_one_control_that_defaults_on() -> None: + """strict_scope is the only control here whose default is on. + + Every other line on the panel defaults to off, where "off (default)" + answers nothing. This one ends a run before the first phase, so "off" + has to say who turned it off. + """ + runner = CliRunner() + assert "Strict scope: on (default)" in runner.invoke(main.cli, ["status"]).output + + +def test_a_variable_that_is_set_is_named_even_when_it_is_empty(monkeypatch) -> None: + """Set-ness is a fact about the environment; the value is a separate one. + + Truthiness here would report an empty variable as the default, which is + the panel answering a question about a value when the question was about + the environment. Measured while adding this line: an empty + CYBERAI_STRICT_SCOPE turns the refusal off, because _env_bool treats an + empty string as a chosen no while _env_int and _env_float treat it as + nobody choosing. The panel has to name the variable in that case or the + operator has no way to see it. + """ + monkeypatch.setenv("CYBERAI_STRICT_SCOPE", "") + output = CliRunner().invoke(main.cli, ["status"]).output + assert "(CYBERAI_STRICT_SCOPE)" in output + + +def test_a_zero_budget_reads_as_disabled_rather_than_as_no_money(monkeypatch) -> None: + monkeypatch.delenv("CYBERAI_MAX_COST_USD", raising=False) + assert "Cost budget: disabled" in CliRunner().invoke(main.cli, ["status"]).output + monkeypatch.setenv("CYBERAI_MAX_COST_USD", "5") + assert "Cost budget: 5.0 USD" in CliRunner().invoke(main.cli, ["status"]).output + + +def test_every_field_the_orchestrator_reads_reaches_the_screen() -> None: + """The screen shows what governs the run, checked against the run. + + Not a hand-written list: the fields are scanned out of orchestrator.py, + so a control added there without a line here fails. Four fields on + CyberAIConfig are read by nobody at all -- intel, timeout, verbose, + use_lab_dogfood -- and a rule written as "every field" would have + demanded a line for a lever that moves nothing. + """ + root = pathlib.Path(__file__).resolve().parents[2] + tree = ast.parse((root / "cyberai" / "core" / "orchestrator.py").read_text()) + read: set[str] = set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Attribute) + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "self" + and node.value.attr == "config" + ): + read.add(node.attr) + + # field on CyberAIConfig -> the label that stands for it on the panel + shown = { + "air_gapped": "Air-gapped", + "enable_planner": "Planner", + "enable_replan": "Replan", + "llm": "Provider", + "max_cost_usd": "Cost budget", + "output_dir": "Output", + "routing": "Model routing", + "strict_scope": "Strict scope", + "use_planned_redteam": "Planned redteam", + "use_web_recon": "Web recon", + } + assert read == set(shown), ( + f"orchestrator reads but the map does not name: {read - set(shown)}; " + f"the map names what the orchestrator no longer reads: {set(shown) - read}" + ) + + output = CliRunner().invoke(main.cli, ["status"]).output + for field, label in shown.items(): + assert f"{label}:" in output, f"{field} governs the run and has no line" + + def test_the_probe_budget_is_shorter_than_the_manifest_budget() -> None: """Three minutes of hung binaries is not an answer to "what is wrong".""" assert env.STATUS_VERSION_TIMEOUT < env.VERSION_TIMEOUT From f85daf6d119e879558a213fb46aa78bca832c7cd Mon Sep 17 00:00:00 2001 From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com> Date: Sun, 20 Sep 2026 10:36:19 +0300 Subject: [PATCH 4/4] fix(config): a typo in the scope variable no longer disarms the refusal Found while printing the new status line: CYBERAI_STRICT_SCOPE= turned the scope refusal off. So did CYBERAI_STRICT_SCOPE=nope. An empty variable is what a shell leaves behind for VAR= in a .env file or VAR=$MISSING in a script, so the guard that risk 20 rests on could come down without anyone deciding anything. The cause is not a bug in _env_bool. It answers "is this value one of the words for yes", and every other value is no -- deliberate, asserted by test_env_bool_falsy_values with the empty string named in the list. For the twenty-two flags that default to off that rule costs nothing: an unrecognised value lands where the default already was. strict_scope is the one flag here whose default is on, so the same rule handed the refusal to any string at all. So the fix is at the point of use, not in the shared reader. _env_guard_bool recognises both lists -- the words for yes and the words for no -- and treats anything else as nobody having chosen, leaving the default standing. That matches what the README table and risk 20 already promised: 0 and --no-strict-scope are the named ways to proceed without a scope. Now they are the only ways. It still does not raise: from_env has no raise in it by design, because garbage in a variable must not abort a scan at startup. README line 298 is rewritten in the same pass. It said "Refuse the exploit phase", which stopped being true on day 50 when the refusal moved ahead of the first phase. Mutation: four mutants, four killed. The fourth changes _env_bool itself the way this commit deliberately did not, and it fails test_env_bool_falsy_values in another file -- the measurement that the old semantics were chosen rather than overlooked. --- README.md | 4 +-- blog/launch-post-draft.md | 2 +- cyberai/core/config.py | 41 ++++++++++++++++++++++++++-- docs/architecture/risk-register.md | 2 +- tests/unit/test_strict_scope.py | 43 +++++++++++++++++++++++++++++- 5 files changed, 85 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 65ac536b..b4438af7 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-2955%20collected-brightgreen) +![Tests](https://img.shields.io/badge/tests-2957%20collected-brightgreen) ![Mypy](https://img.shields.io/badge/mypy-strict%3A%20100%2F172%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) @@ -295,7 +295,7 @@ Every setting can be driven from the environment (or a `.env` file - see | `CYBERAI_ENABLE_REPLAN` | Critic-driven phase replan | | `CYBERAI_USE_EXPLOIT_MEMORY` | Recall similar past exploit chains | | `CYBERAI_AIR_GAPPED` | Force local-only (no-egress) LLM path | -| `CYBERAI_STRICT_SCOPE` | Refuse the exploit phase when no scope was given (default on; `0` or `--no-strict-scope` to opt out) | +| `CYBERAI_STRICT_SCOPE` | Refuse an unauthorised run before the first phase (default on; `0`, `false`, `no`, `off` or `--no-strict-scope` to opt out -- any other value leaves the refusal on) | | `CYBERAI_ENABLE_MODEL_ROUTING` | Per-phase model selection | | `CYBERAI_MAX_COST_USD` | LLM spend budget (0 = disabled) | | `CYBERAI_OUTPUT_DIR` | Report output directory | diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index 6ba4752a..16bd7cd8 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,7 +13,7 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -2955 tests collected under the gated selection run before every commit, with the +2957 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` clean over 100 of 172 modules, Apache-2.0. diff --git a/cyberai/core/config.py b/cyberai/core/config.py index f3dbdc7d..826cdda2 100644 --- a/cyberai/core/config.py +++ b/cyberai/core/config.py @@ -32,7 +32,44 @@ def _env_bool(name: str, default: bool) -> bool: raw = os.getenv(name) if raw is None: return default - return raw.strip().lower() in {"1", "true", "yes", "on"} + return raw.strip().lower() in _TRUTHY + + +# The words that turn a flag on and the words that turn it off. _env_bool +# reads anything outside the first set as off, which is harmless for a flag +# defaulting to off -- an unrecognised value lands on the default anyway. +_TRUTHY = frozenset({"1", "true", "yes", "on"}) +_FALSY = frozenset({"0", "false", "no", "off"}) + + +def _env_guard_bool(name: str, default: bool) -> bool: + """Read a flag whose default protects the run, where a typo must not disarm it. + + _env_bool answers the question "is this value one of the words for yes", + so every other value, including an empty string and a misspelling, reads + as no. For the twenty-two flags that default to off that is the same + answer as the default and costs nothing. For a flag that defaults to on + it is the difference between a guard and no guard: measured on + 2026-09-19, CYBERAI_STRICT_SCOPE= (empty, the shape a shell leaves behind + for an unset variable in a .env file or a VAR=$MISSING expansion) turned + the scope refusal off, and so did CYBERAI_STRICT_SCOPE=nope. + + The documented ways to proceed without a scope are named ones -- `0` and + --no-strict-scope, per the README table and risk 20 in the register -- + and this reader holds to exactly that list. A value outside both sets is + nobody having chosen, not a choice of off, so the default stands. It + still does not raise: from_env has no raise in it by design, because + garbage in a variable must not abort a scan at startup. + """ + raw = os.getenv(name) + if raw is None: + return default + value = raw.strip().lower() + if value in _TRUTHY: + return True + if value in _FALSY: + return False + return default def _env_float(name: str, default: float) -> float: @@ -307,5 +344,5 @@ def from_env(cls) -> "CyberAIConfig": use_lab_dogfood=_env_bool("CYBERAI_USE_LAB_DOGFOOD", False), web_enable_bench_trigger=_env_bool("CYBERAI_WEB_ENABLE_BENCH_TRIGGER", False), air_gapped=_env_bool("CYBERAI_AIR_GAPPED", False), - strict_scope=_env_bool("CYBERAI_STRICT_SCOPE", True), + strict_scope=_env_guard_bool("CYBERAI_STRICT_SCOPE", True), ) diff --git a/docs/architecture/risk-register.md b/docs/architecture/risk-register.md index cbe20cb4..e839dd2a 100644 --- a/docs/architecture/risk-register.md +++ b/docs/architecture/risk-register.md @@ -58,7 +58,7 @@ not fixed. `unguarded` — held by discipline, with no machine behind it. | # | Risk | Status | Measured by | |---|---|---|---| | 19 | Publishing numbers produced by a path that was broken | closed | `tests/architecture/test_nothing_publishes_without_the_checks.py::test_the_upload_cannot_run_before_the_guard` | -| 20 | An offensive tool that does not require an authorisation scope | closed | `tests/unit/test_the_refusal_comes_before_the_first_packet.py::test_an_unscoped_run_never_reaches_the_recon_agent` holds the refusal ahead of the pipeline: measured on 2026-09-17, an unscoped run against a protected range had already spent 51 seconds on nmap, whois, dns and subdomain enumeration before the exploit phase declined it. The row said closed while the target was being touched, because the test behind it read a config field rather than the network. `--no-strict-scope` and `CYBERAI_STRICT_SCOPE=0` remain the named ways to proceed anyway | +| 20 | An offensive tool that does not require an authorisation scope | closed | `tests/unit/test_the_refusal_comes_before_the_first_packet.py::test_an_unscoped_run_never_reaches_the_recon_agent` holds the refusal ahead of the pipeline: measured on 2026-09-17, an unscoped run against a protected range had already spent 51 seconds on nmap, whois, dns and subdomain enumeration before the exploit phase declined it. The row said closed while the target was being touched, because the test behind it read a config field rather than the network. `--no-strict-scope` and `CYBERAI_STRICT_SCOPE=0` remain the named ways to proceed anyway, and since 2026-09-19 they are the only ones: `tests/unit/test_strict_scope.py::test_only_a_named_word_turns_the_refusal_off` holds the refusal against a value nobody chose. The flag reader answers "is this one of the words for yes", so an empty variable -- what a shell leaves for `VAR=` in a .env file -- and a misspelling both read as off, which disarmed the one flag here whose default protects the run | | 21 | Private plans and journals reaching a public repository | unguarded | no test and no workflow step checks this. A tree-wide history search finds none of those filenames, which is a measurement of the past, not a guard on the next commit | | 22 | One developer, measuring by hand | open | structural. The mitigation is that every closed row above names a test rather than a memory | | 23 | A measurement that cannot tell a quiet environment from a clean tree | partly | `tests/architecture/test_the_drift_report_refuses_an_unmeasured_environment.py::test_a_checker_that_never_ran_is_not_a_clean_package` and the four rows beside it. Measured on 2026-09-18 on an untouched checkout: with no mypy installed the drift report printed 172 clean modules, 0 errors and no drift and exited zero; without `types-networkx` it printed 275 errors and accused `cyberai/core/kb_graph.py` of being undeclared; on mcp 1.28.1 it printed 284 instead of 285. Five signals, one tree, every guard green, because the guards read declarations rather than the environment. The report now refuses a run it cannot vouch for and names the package that differs. Partly: the same question is unanswered for the test suite, which collected nothing on a machine without `pytest-asyncio` and said so as three collection errors rather than as an unmeasured environment | diff --git a/tests/unit/test_strict_scope.py b/tests/unit/test_strict_scope.py index 04975c9a..6071c163 100644 --- a/tests/unit/test_strict_scope.py +++ b/tests/unit/test_strict_scope.py @@ -25,7 +25,7 @@ import pytest from cyberai.agents.exploit.safety_validator import validate_exploit_scope -from cyberai.core.config import CyberAIConfig +from cyberai.core.config import CyberAIConfig, _env_bool, _env_guard_bool def test_empty_scope_warns_by_default() -> None: @@ -73,6 +73,47 @@ def test_config_field_defaults_on_and_reads_the_environment( assert CyberAIConfig.from_env().strict_scope is False +def test_only_a_named_word_turns_the_refusal_off( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A typo is not consent. + + _env_bool reads every value outside its truthy set as false, which is the + right answer for a flag that defaults to off -- an unrecognised value + lands where the default already was. This flag defaults to on, so the + same rule handed the refusal to anything at all: measured on 2026-09-19, + CYBERAI_STRICT_SCOPE= turned it off, and so did CYBERAI_STRICT_SCOPE=nope. + An empty variable is what a shell leaves behind for VAR= in a .env file + or VAR=$MISSING in a script, so this was reachable without anyone + deciding anything. + + The README table and risk 20 both name the ways to proceed without a + scope: `0` and --no-strict-scope. This asserts that list and no more. + """ + for word in ("0", "false", "no", "off", "OFF", " 0 "): + monkeypatch.setenv("CYBERAI_STRICT_SCOPE", word) + assert CyberAIConfig.from_env().strict_scope is False, word + + for noise in ("", " ", "nope", "flase", "2", "null", "none"): + monkeypatch.setenv("CYBERAI_STRICT_SCOPE", noise) + assert CyberAIConfig.from_env().strict_scope is True, noise + + +def test_the_ordinary_flag_reader_is_left_as_it_was( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The guard reader is for guards; the other twenty-two keep their reader. + + Changing _env_bool itself would have rewritten the meaning of every flag + in the file to fix one of them. For a flag defaulting to off the two + readers cannot disagree, but the distinction is the point, so it is + stated here rather than left to be rediscovered. + """ + monkeypatch.setenv("CYBERAI_TEST_FLAG", "nope") + assert _env_bool("CYBERAI_TEST_FLAG", True) is False + assert _env_guard_bool("CYBERAI_TEST_FLAG", True) is True + + def test_the_orchestrator_hands_the_flag_to_the_validator() -> None: """The link the other tests cannot see.