diff --git a/README.md b/README.md index b9b9b104..b4438af7 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-2946%20collected-brightgreen) +![Tests](https://img.shields.io/badge/tests-2957%20collected-brightgreen) ![Mypy](https://img.shields.io/badge/mypy-strict%3A%20100%2F172%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) @@ -76,7 +76,7 @@ cyberai scan example.com --dry-run # real scan with a local model (air-gapped, no cloud) and scope cyberai scan app.target.com --provider ollama --scope "*.target.com" -cyberai status # provider, output, trust boundary, sampling, air-gap, credentials, toolchain +cyberai status # provider, output, trust boundary, sampling, air-gap, credentials, toolchain, run controls cyberai replay # re-run a saved session ``` @@ -295,7 +295,7 @@ Every setting can be driven from the environment (or a `.env` file - see | `CYBERAI_ENABLE_REPLAN` | Critic-driven phase replan | | `CYBERAI_USE_EXPLOIT_MEMORY` | Recall similar past exploit chains | | `CYBERAI_AIR_GAPPED` | Force local-only (no-egress) LLM path | -| `CYBERAI_STRICT_SCOPE` | Refuse the exploit phase when no scope was given (default on; `0` or `--no-strict-scope` to opt out) | +| `CYBERAI_STRICT_SCOPE` | Refuse an unauthorised run before the first phase (default on; `0`, `false`, `no`, `off` or `--no-strict-scope` to opt out -- any other value leaves the refusal on) | | `CYBERAI_ENABLE_MODEL_ROUTING` | Per-phase model selection | | `CYBERAI_MAX_COST_USD` | LLM spend budget (0 = disabled) | | `CYBERAI_OUTPUT_DIR` | Report output directory | diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index 0bf8cf60..16bd7cd8 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,7 +13,7 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -2946 tests collected under the gated selection run before every commit, with the +2957 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` clean over 100 of 172 modules, Apache-2.0. diff --git a/cyberai/__main__.py b/cyberai/__main__.py index 01c2f4b2..ff257284 100644 --- a/cyberai/__main__.py +++ b/cyberai/__main__.py @@ -10,15 +10,7 @@ from cyberai.version import __version__ -from .agents.exploit.nuclei_engine import find_nuclei -from .agents.exploit.searchsploit import find_searchsploit -from .agents.mcp_scan.mst_bridge import find_mst -from .agents.recon.nmap_tool import find_nmap -from .agents.web3.aderyn_tool import find_aderyn -from .agents.web3.anvil_harness import find_anvil -from .agents.web3.foundry_poc import find_forge -from .agents.web3.halmos_tool import find_halmos -from .agents.web3.slither_tool import find_slither +from .bench.environment import STATUS_VERSION_TIMEOUT, TOOL_PROBES, probe_toolchain from .cli.bench import bench from .cli.detector_eval import detector from .cli.mcp_scan import mcp_scan @@ -33,17 +25,15 @@ # binary name -> the resolver that already locates it for the agents. Not a # second lookup: status calls the same functions the tools call, so a name # reported here is the one that will be executed. -_TOOLCHAIN = { - "nmap": find_nmap, - "nuclei": find_nuclei, - "searchsploit": find_searchsploit, - "forge": find_forge, - "aderyn": find_aderyn, - "slither": find_slither, - "anvil": find_anvil, - "halmos": find_halmos, - "mas-sentry": find_mst, -} +# +# Derived from the probe registry rather than written out a second time. The +# two copies had drifted: this one said "mas-sentry", which is the string the +# finder hands to shutil.which, while the probe registry said "mst" and wrote +# that name into every run manifest. A toolchain drift was reported against a +# binary no resolver would ever find. With one registry the two guards over +# this display -- that it names every finder in the package, and that each +# label is the binary its finder looks for -- reach the manifest as well. +_TOOLCHAIN = {probe.name: probe.resolver for probe in TOOL_PROBES} def _detach_stdin_from_tty() -> None: @@ -522,21 +512,79 @@ def _seed_line(llm: LLMConfig) -> str: return str(llm.seed) -def _toolchain_lines() -> tuple[str, str]: +def _toolchain_lines(*, versions: bool = False) -> tuple[str, str]: """Which external binaries resolve right now, both sides named. Both halves are sets of names rather than counts: "7 of 8 found" would say nothing about which one is missing, and a run that needs the missing one fails later with no hint that status already knew. + + ``versions`` runs each located binary's version flag. Off by default + because this resolves paths and starts no process, while a probe spends + up to STATUS_VERSION_TIMEOUT on every tool that answers slowly -- and + status is the command an operator reaches for when something is already + wrong. The same split the bench CLI makes, which imports the probe only + when a manifest is asked for. + + Under ``versions`` both halves come out of one probe pass rather than a + second walk over the resolvers: two walks can disagree, which is the + shape of defect that had the display and the manifest naming the same + binary differently. """ - found = [name for name, finder in _TOOLCHAIN.items() if finder() is not None] - missing = [name for name in _TOOLCHAIN if name not in found] + if not versions: + found = [name for name, finder in _TOOLCHAIN.items() if finder() is not None] + missing = [name for name in _TOOLCHAIN if name not in found] + return ( + ", ".join(found) if found else "none", + ", ".join(missing) if missing else "none", + ) + + probed = probe_toolchain(timeout=STATUS_VERSION_TIMEOUT) + found = [ + f"{t.name} {t.version}" if t.version else f"{t.name} ({t.detail})" + for t in probed + if t.path is not None + ] + missing = [t.name for t in probed if t.path is None] return ( ", ".join(found) if found else "none", ", ".join(missing) if missing else "none", ) +def _budget_line(config: CyberAIConfig) -> str: + """The LLM spend ceiling. Zero disables the check rather than forbidding + spend, so it is spelled out: a panel that printed "0.0" would read as a + budget of nothing.""" + if config.max_cost_usd <= 0: + return "disabled" + return f"{config.max_cost_usd} USD" + + +def _strict_scope_line(config: CyberAIConfig) -> str: + """Whether the run refuses an unauthorised target, and who decided. + + The only line on this panel that names its source. strict_scope is the + one control here that defaults to on, and the one whose value ends a run + before the first phase, so "off" has to answer who turned it off. The + other controls default to off, and "(default)" against six of them would + be noise rather than an answer. + + The source is read from the environment rather than inferred from the + value: every reader in config.py returns the default for an unset + variable and for an unparseable one alike, so the value cannot say which + happened. What is stated here is what can be checked -- the variable is + set, or it is not. + """ + state = "on" if config.strict_scope else "off" + # `is None`, not truthiness: an empty variable is set, and reporting it as + # the default would be the panel answering about a value when the question + # is about the environment. + set_here = os.getenv("CYBERAI_STRICT_SCOPE") is not None + source = "CYBERAI_STRICT_SCOPE" if set_here else "default" + return f"{state} ({source})" + + def _api_key_line(llm: LLMConfig) -> str: """Whether the credential this provider needs is present. @@ -552,7 +600,12 @@ def _api_key_line(llm: LLMConfig) -> str: @cli.command() -def status() -> None: +@click.option( + "--versions", + is_flag=True, + help="Run each located tool's version flag (starts processes)", +) +def status(versions: bool) -> None: """Show CyberAI status and config.""" config = CyberAIConfig.from_env() # The guard a real client would build from this config, not a second @@ -560,7 +613,7 @@ def status() -> None: # value; printing the raw setting would name a policy that never acts. guard = LLMClient(config.llm).guard classifier = guard.classifier - tools_found, tools_missing = _toolchain_lines() + tools_found, tools_missing = _toolchain_lines(versions=versions) console.print( Panel( f"Provider: {config.llm.provider}\n" @@ -574,7 +627,14 @@ def status() -> None: f"Air-gapped: {'on' if config.air_gapped else 'off'}\n" f"API key: {_api_key_line(config.llm)}\n" f"Tools found: {tools_found}\n" - f"Tools missing: {tools_missing}", + f"Tools missing: {tools_missing}\n" + f"Strict scope: {_strict_scope_line(config)}\n" + f"Cost budget: {_budget_line(config)}\n" + f"Planner: {'on' if config.enable_planner else 'off'}\n" + f"Replan: {'on' if config.enable_replan else 'off'}\n" + f"Model routing: {'on' if config.routing.enable_model_routing else 'off'}\n" + f"Web recon: {'on' if config.use_web_recon else 'off'}\n" + f"Planned redteam: {'on' if config.use_planned_redteam else 'off'}", title="CyberAI Status", ) ) diff --git a/cyberai/bench/environment.py b/cyberai/bench/environment.py index 3f88386d..b114688b 100644 --- a/cyberai/bench/environment.py +++ b/cyberai/bench/environment.py @@ -53,6 +53,14 @@ VERSION_TIMEOUT = 20 +# Shorter than VERSION_TIMEOUT, and the difference is the caller, not the tool. +# A benchmark manifest is written once and has to record a version even from a +# tool that starts slowly. `status` is the command an operator runs when +# something is already wrong, and nine hung binaries at VERSION_TIMEOUT would +# hold that answer for three minutes. Measured: a binary that never answers +# costs exactly the timeout, 20.02 s at the default. +STATUS_VERSION_TIMEOUT = 3 + NOT_INSTALLED = "not installed" FLAG_NOT_MEASURED = "version flag not measured" NOT_PARSED = "version not parsed" @@ -82,7 +90,7 @@ class ToolProbe: ToolProbe("forge", find_forge, "--version"), ToolProbe("anvil", find_anvil, "--version"), ToolProbe("halmos", find_halmos, "--version"), - ToolProbe("mst", find_mst, None), + ToolProbe("mas-sentry", find_mst, None), ) @@ -107,15 +115,19 @@ def parse_version(stdout: str, stderr: str) -> str | None: return None -def probe_tool(probe: ToolProbe) -> ToolVersion: - """Resolve one binary and read its version. Never raises.""" +def probe_tool(probe: ToolProbe, *, timeout: float = VERSION_TIMEOUT) -> ToolVersion: + """Resolve one binary and read its version. Never raises. + + ``timeout`` is keyword-only and defaults to the manifest's budget, so the + callers that were here before this parameter existed read the same way. + """ path = probe.resolver() if path is None: return ToolVersion(probe.name, None, None, NOT_INSTALLED) if probe.flag is None: return ToolVersion(probe.name, path, None, FLAG_NOT_MEASURED) try: - proc = run_sealed([path, probe.flag], timeout=VERSION_TIMEOUT) + proc = run_sealed([path, probe.flag], timeout=timeout) except (subprocess.SubprocessError, OSError) as exc: return ToolVersion(probe.name, path, None, f"probe failed: {type(exc).__name__}") version = parse_version(proc.stdout, proc.stderr) @@ -124,11 +136,13 @@ def probe_tool(probe: ToolProbe) -> ToolVersion: return ToolVersion(probe.name, path, version, "") -def probe_toolchain(probes: Sequence[ToolProbe] = TOOL_PROBES) -> tuple[ToolVersion, ...]: +def probe_toolchain( + probes: Sequence[ToolProbe] = TOOL_PROBES, *, timeout: float = VERSION_TIMEOUT +) -> tuple[ToolVersion, ...]: """Probe every binary the platform drives, in registry order. A tuple, not a list: the result is stored on a frozen RunManifest and fingerprinted, and a mutable sequence there would be a shared reference into a record that claims to be immutable. """ - return tuple(probe_tool(probe) for probe in probes) + return tuple(probe_tool(probe, timeout=timeout) for probe in probes) diff --git a/cyberai/core/config.py b/cyberai/core/config.py index f3dbdc7d..826cdda2 100644 --- a/cyberai/core/config.py +++ b/cyberai/core/config.py @@ -32,7 +32,44 @@ def _env_bool(name: str, default: bool) -> bool: raw = os.getenv(name) if raw is None: return default - return raw.strip().lower() in {"1", "true", "yes", "on"} + return raw.strip().lower() in _TRUTHY + + +# The words that turn a flag on and the words that turn it off. _env_bool +# reads anything outside the first set as off, which is harmless for a flag +# defaulting to off -- an unrecognised value lands on the default anyway. +_TRUTHY = frozenset({"1", "true", "yes", "on"}) +_FALSY = frozenset({"0", "false", "no", "off"}) + + +def _env_guard_bool(name: str, default: bool) -> bool: + """Read a flag whose default protects the run, where a typo must not disarm it. + + _env_bool answers the question "is this value one of the words for yes", + so every other value, including an empty string and a misspelling, reads + as no. For the twenty-two flags that default to off that is the same + answer as the default and costs nothing. For a flag that defaults to on + it is the difference between a guard and no guard: measured on + 2026-09-19, CYBERAI_STRICT_SCOPE= (empty, the shape a shell leaves behind + for an unset variable in a .env file or a VAR=$MISSING expansion) turned + the scope refusal off, and so did CYBERAI_STRICT_SCOPE=nope. + + The documented ways to proceed without a scope are named ones -- `0` and + --no-strict-scope, per the README table and risk 20 in the register -- + and this reader holds to exactly that list. A value outside both sets is + nobody having chosen, not a choice of off, so the default stands. It + still does not raise: from_env has no raise in it by design, because + garbage in a variable must not abort a scan at startup. + """ + raw = os.getenv(name) + if raw is None: + return default + value = raw.strip().lower() + if value in _TRUTHY: + return True + if value in _FALSY: + return False + return default def _env_float(name: str, default: float) -> float: @@ -307,5 +344,5 @@ def from_env(cls) -> "CyberAIConfig": use_lab_dogfood=_env_bool("CYBERAI_USE_LAB_DOGFOOD", False), web_enable_bench_trigger=_env_bool("CYBERAI_WEB_ENABLE_BENCH_TRIGGER", False), air_gapped=_env_bool("CYBERAI_AIR_GAPPED", False), - strict_scope=_env_bool("CYBERAI_STRICT_SCOPE", True), + strict_scope=_env_guard_bool("CYBERAI_STRICT_SCOPE", True), ) diff --git a/docs/architecture/risk-register.md b/docs/architecture/risk-register.md index cbe20cb4..e839dd2a 100644 --- a/docs/architecture/risk-register.md +++ b/docs/architecture/risk-register.md @@ -58,7 +58,7 @@ not fixed. `unguarded` — held by discipline, with no machine behind it. | # | Risk | Status | Measured by | |---|---|---|---| | 19 | Publishing numbers produced by a path that was broken | closed | `tests/architecture/test_nothing_publishes_without_the_checks.py::test_the_upload_cannot_run_before_the_guard` | -| 20 | An offensive tool that does not require an authorisation scope | closed | `tests/unit/test_the_refusal_comes_before_the_first_packet.py::test_an_unscoped_run_never_reaches_the_recon_agent` holds the refusal ahead of the pipeline: measured on 2026-09-17, an unscoped run against a protected range had already spent 51 seconds on nmap, whois, dns and subdomain enumeration before the exploit phase declined it. The row said closed while the target was being touched, because the test behind it read a config field rather than the network. `--no-strict-scope` and `CYBERAI_STRICT_SCOPE=0` remain the named ways to proceed anyway | +| 20 | An offensive tool that does not require an authorisation scope | closed | `tests/unit/test_the_refusal_comes_before_the_first_packet.py::test_an_unscoped_run_never_reaches_the_recon_agent` holds the refusal ahead of the pipeline: measured on 2026-09-17, an unscoped run against a protected range had already spent 51 seconds on nmap, whois, dns and subdomain enumeration before the exploit phase declined it. The row said closed while the target was being touched, because the test behind it read a config field rather than the network. `--no-strict-scope` and `CYBERAI_STRICT_SCOPE=0` remain the named ways to proceed anyway, and since 2026-09-19 they are the only ones: `tests/unit/test_strict_scope.py::test_only_a_named_word_turns_the_refusal_off` holds the refusal against a value nobody chose. The flag reader answers "is this one of the words for yes", so an empty variable -- what a shell leaves for `VAR=` in a .env file -- and a misspelling both read as off, which disarmed the one flag here whose default protects the run | | 21 | Private plans and journals reaching a public repository | unguarded | no test and no workflow step checks this. A tree-wide history search finds none of those filenames, which is a measurement of the past, not a guard on the next commit | | 22 | One developer, measuring by hand | open | structural. The mitigation is that every closed row above names a test rather than a memory | | 23 | A measurement that cannot tell a quiet environment from a clean tree | partly | `tests/architecture/test_the_drift_report_refuses_an_unmeasured_environment.py::test_a_checker_that_never_ran_is_not_a_clean_package` and the four rows beside it. Measured on 2026-09-18 on an untouched checkout: with no mypy installed the drift report printed 172 clean modules, 0 errors and no drift and exited zero; without `types-networkx` it printed 275 errors and accused `cyberai/core/kb_graph.py` of being undeclared; on mcp 1.28.1 it printed 284 instead of 285. Five signals, one tree, every guard green, because the guards read declarations rather than the environment. The report now refuses a run it cannot vouch for and names the package that differs. Partly: the same question is unanswered for the test suite, which collected nothing on a machine without `pytest-asyncio` and said so as three collection errors rather than as an unmeasured environment | diff --git a/tests/architecture/test_the_readme_names_what_status_prints.py b/tests/architecture/test_the_readme_names_what_status_prints.py index e8c04c9b..ed97b11d 100644 --- a/tests/architecture/test_the_readme_names_what_status_prints.py +++ b/tests/architecture/test_the_readme_names_what_status_prints.py @@ -26,6 +26,15 @@ "air-gap": ["Air-gapped"], "credentials": ["API key"], "toolchain": ["Tools found", "Tools missing"], + "run controls": [ + "Strict scope", + "Cost budget", + "Planner", + "Replan", + "Model routing", + "Web recon", + "Planned redteam", + ], } # Both files are located from this file rather than from the imported diff --git a/tests/unit/test_bench_environment.py b/tests/unit/test_bench_environment.py index f4bd3cb1..232381b8 100644 --- a/tests/unit/test_bench_environment.py +++ b/tests/unit/test_bench_environment.py @@ -102,7 +102,7 @@ def test_every_probe_resolves_through_the_module_that_owns_the_tool(): "forge": ("cyberai.agents.web3.foundry_poc", "find_forge"), "anvil": ("cyberai.agents.web3.anvil_harness", "find_anvil"), "halmos": ("cyberai.agents.web3.halmos_tool", "find_halmos"), - "mst": ("cyberai.agents.mcp_scan.mst_bridge", "find_mst"), + "mas-sentry": ("cyberai.agents.mcp_scan.mst_bridge", "find_mst"), } got = {p.name: (p.resolver.__module__, p.resolver.__name__) for p in env.TOOL_PROBES} assert got == expected diff --git a/tests/unit/test_status_versions.py b/tests/unit/test_status_versions.py new file mode 100644 index 00000000..cb6d0a23 --- /dev/null +++ b/tests/unit/test_status_versions.py @@ -0,0 +1,173 @@ +"""`status --versions` asks the tools, and plain `status` asks nothing. + +Two halves of one contract. The flag exists because a probe starts a process +per located tool and status is the command an operator runs when something is +already wrong; a default that pays that cost would be the defect the flag was +added to avoid. So the cheap path has to stay provably cheap: the assertion is +that no process starts, not that the output looks the same. +""" + +from __future__ import annotations + +import ast +import pathlib + +from click.testing import CliRunner + +from cyberai import __main__ as main +from cyberai.bench import environment as env +from cyberai.bench.run_manifest import ToolVersion + + +def test_plain_status_starts_no_process(tmp_path, monkeypatch) -> None: + """A tool has to be found before "nothing ran" means anything. + + The first version of this test patched run_sealed on a machine with no + tools installed: the probe returned "not installed" before reaching any + process, so the assertion held whatever the code did. Mutating the flag + to probe unconditionally left it green. The stub below makes the binary + resolvable, so a probe would reach run_sealed and the assertion has + something to fail on. + """ + stub = tmp_path / "nmap" + stub.write_text("#!/bin/sh\necho 'Nmap version 7.98 ( https://nmap.org )'\n") + stub.chmod(0o755) + monkeypatch.setenv("NMAP_PATH", str(stub)) + + def boom(*args: object, **kwargs: object) -> object: + raise AssertionError("status ran a binary without --versions") + + monkeypatch.setattr(env, "run_sealed", boom) + result = CliRunner().invoke(main.cli, ["status"]) + assert result.exit_code == 0 + assert "nmap" in result.output + + +def test_the_flag_reports_the_version_the_tool_prints(tmp_path, monkeypatch) -> None: + stub = tmp_path / "nmap" + stub.write_text("#!/bin/sh\necho 'Nmap version 7.98 ( https://nmap.org )'\n") + stub.chmod(0o755) + monkeypatch.setenv("NMAP_PATH", str(stub)) + result = CliRunner().invoke(main.cli, ["status", "--versions"]) + assert "nmap 7.98" in result.output + + +def test_a_located_tool_with_no_flag_says_so_rather_than_guessing(tmp_path, monkeypatch) -> None: + """searchsploit and mas-sentry carry flag=None: no version was ever measured.""" + stub = tmp_path / "searchsploit" + stub.write_text("#!/bin/sh\necho 'Usage: searchsploit'\n") + stub.chmod(0o755) + monkeypatch.setenv("SEARCHSPLOIT_PATH", str(stub)) + result = CliRunner().invoke(main.cli, ["status", "--versions"]) + assert f"searchsploit ({env.FLAG_NOT_MEASURED})" in result.output + + +def test_the_screen_names_the_source_of_the_one_control_that_defaults_on() -> None: + """strict_scope is the only control here whose default is on. + + Every other line on the panel defaults to off, where "off (default)" + answers nothing. This one ends a run before the first phase, so "off" + has to say who turned it off. + """ + runner = CliRunner() + assert "Strict scope: on (default)" in runner.invoke(main.cli, ["status"]).output + + +def test_a_variable_that_is_set_is_named_even_when_it_is_empty(monkeypatch) -> None: + """Set-ness is a fact about the environment; the value is a separate one. + + Truthiness here would report an empty variable as the default, which is + the panel answering a question about a value when the question was about + the environment. Measured while adding this line: an empty + CYBERAI_STRICT_SCOPE turns the refusal off, because _env_bool treats an + empty string as a chosen no while _env_int and _env_float treat it as + nobody choosing. The panel has to name the variable in that case or the + operator has no way to see it. + """ + monkeypatch.setenv("CYBERAI_STRICT_SCOPE", "") + output = CliRunner().invoke(main.cli, ["status"]).output + assert "(CYBERAI_STRICT_SCOPE)" in output + + +def test_a_zero_budget_reads_as_disabled_rather_than_as_no_money(monkeypatch) -> None: + monkeypatch.delenv("CYBERAI_MAX_COST_USD", raising=False) + assert "Cost budget: disabled" in CliRunner().invoke(main.cli, ["status"]).output + monkeypatch.setenv("CYBERAI_MAX_COST_USD", "5") + assert "Cost budget: 5.0 USD" in CliRunner().invoke(main.cli, ["status"]).output + + +def test_every_field_the_orchestrator_reads_reaches_the_screen() -> None: + """The screen shows what governs the run, checked against the run. + + Not a hand-written list: the fields are scanned out of orchestrator.py, + so a control added there without a line here fails. Four fields on + CyberAIConfig are read by nobody at all -- intel, timeout, verbose, + use_lab_dogfood -- and a rule written as "every field" would have + demanded a line for a lever that moves nothing. + """ + root = pathlib.Path(__file__).resolve().parents[2] + tree = ast.parse((root / "cyberai" / "core" / "orchestrator.py").read_text()) + read: set[str] = set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Attribute) + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "self" + and node.value.attr == "config" + ): + read.add(node.attr) + + # field on CyberAIConfig -> the label that stands for it on the panel + shown = { + "air_gapped": "Air-gapped", + "enable_planner": "Planner", + "enable_replan": "Replan", + "llm": "Provider", + "max_cost_usd": "Cost budget", + "output_dir": "Output", + "routing": "Model routing", + "strict_scope": "Strict scope", + "use_planned_redteam": "Planned redteam", + "use_web_recon": "Web recon", + } + assert read == set(shown), ( + f"orchestrator reads but the map does not name: {read - set(shown)}; " + f"the map names what the orchestrator no longer reads: {set(shown) - read}" + ) + + output = CliRunner().invoke(main.cli, ["status"]).output + for field, label in shown.items(): + assert f"{label}:" in output, f"{field} governs the run and has no line" + + +def test_the_probe_budget_is_shorter_than_the_manifest_budget() -> None: + """Three minutes of hung binaries is not an answer to "what is wrong".""" + assert env.STATUS_VERSION_TIMEOUT < env.VERSION_TIMEOUT + + +def test_one_pass_decides_both_halves(monkeypatch) -> None: + """Found and missing come from the same probe result, not two walks. + + Two walks over the resolvers can disagree, and a display that disagrees + with itself is the defect this file's sibling commit removed between the + CLI and the run manifest. + """ + resolved: list[str] = [] + probed: list[str] = [] + + def probe(probes=env.TOOL_PROBES, **kwargs: object): # type: ignore[no-untyped-def] + probed.append("once") + return tuple(ToolVersion(p.name, "/somewhere/" + p.name, "1.0", "") for p in probes) + + monkeypatch.setattr(main, "probe_toolchain", probe) + monkeypatch.setattr( + main, + "_TOOLCHAIN", + {name: (lambda n=name: resolved.append(n) or None) for name in dict(main._TOOLCHAIN)}, + ) + result = CliRunner().invoke(main.cli, ["status", "--versions"]) + + assert probed == ["once"] + assert resolved == [], f"a second walk asked the resolvers again: {resolved}" + assert "Tools missing: none" in result.output diff --git a/tests/unit/test_strict_scope.py b/tests/unit/test_strict_scope.py index 04975c9a..6071c163 100644 --- a/tests/unit/test_strict_scope.py +++ b/tests/unit/test_strict_scope.py @@ -25,7 +25,7 @@ import pytest from cyberai.agents.exploit.safety_validator import validate_exploit_scope -from cyberai.core.config import CyberAIConfig +from cyberai.core.config import CyberAIConfig, _env_bool, _env_guard_bool def test_empty_scope_warns_by_default() -> None: @@ -73,6 +73,47 @@ def test_config_field_defaults_on_and_reads_the_environment( assert CyberAIConfig.from_env().strict_scope is False +def test_only_a_named_word_turns_the_refusal_off( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A typo is not consent. + + _env_bool reads every value outside its truthy set as false, which is the + right answer for a flag that defaults to off -- an unrecognised value + lands where the default already was. This flag defaults to on, so the + same rule handed the refusal to anything at all: measured on 2026-09-19, + CYBERAI_STRICT_SCOPE= turned it off, and so did CYBERAI_STRICT_SCOPE=nope. + An empty variable is what a shell leaves behind for VAR= in a .env file + or VAR=$MISSING in a script, so this was reachable without anyone + deciding anything. + + The README table and risk 20 both name the ways to proceed without a + scope: `0` and --no-strict-scope. This asserts that list and no more. + """ + for word in ("0", "false", "no", "off", "OFF", " 0 "): + monkeypatch.setenv("CYBERAI_STRICT_SCOPE", word) + assert CyberAIConfig.from_env().strict_scope is False, word + + for noise in ("", " ", "nope", "flase", "2", "null", "none"): + monkeypatch.setenv("CYBERAI_STRICT_SCOPE", noise) + assert CyberAIConfig.from_env().strict_scope is True, noise + + +def test_the_ordinary_flag_reader_is_left_as_it_was( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The guard reader is for guards; the other twenty-two keep their reader. + + Changing _env_bool itself would have rewritten the meaning of every flag + in the file to fix one of them. For a flag defaulting to off the two + readers cannot disagree, but the distinction is the point, so it is + stated here rather than left to be rediscovered. + """ + monkeypatch.setenv("CYBERAI_TEST_FLAG", "nope") + assert _env_bool("CYBERAI_TEST_FLAG", True) is False + assert _env_guard_bool("CYBERAI_TEST_FLAG", True) is True + + def test_the_orchestrator_hands_the_flag_to_the_validator() -> None: """The link the other tests cannot see.