diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e64415f2..f349735f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,13 @@ both halves move when modules are added. It has its own tool: python3 scripts/mypy_badge.py +Adding a module to `[tool.mypy] files` costs its own errors and moves the two +boundary counters the scope page states, sometimes in directions the module's +shape does not predict. Read the price before paying it, with the declared +flags and without editing the manifest: + +python3 scripts/scope_price.py cyberai/core/cache.py + ## Lint ruff check cyberai/ --fix diff --git a/README.md b/README.md index 85d25714..67c3039c 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-2983%20collected-brightgreen) -![Mypy](https://img.shields.io/badge/mypy-strict%3A%20104%2F172%20modules-blue) +![Tests](https://img.shields.io/badge/tests-2988%20collected-brightgreen) +![Mypy](https://img.shields.io/badge/mypy-strict%3A%20107%2F172%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index 8017294a..22663f6d 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,9 +13,9 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -2983 tests collected under the gated selection run before every commit, with the +2988 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` -clean over 104 of 172 modules, Apache-2.0. +clean over 107 of 172 modules, Apache-2.0. It is not a wrapper that pipes nmap output into a chat model. Three things make it a different category of tool. diff --git a/cyberai/agents/web3/immunefi_severity.py b/cyberai/agents/web3/immunefi_severity.py index b4e09eb0..985d7418 100644 --- a/cyberai/agents/web3/immunefi_severity.py +++ b/cyberai/agents/web3/immunefi_severity.py @@ -15,7 +15,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Any, Dict, List if TYPE_CHECKING: from .slither_tool import SlitherFinding @@ -147,7 +147,7 @@ def classify(finding: "SlitherFinding") -> str: return _IMPACT_FALLBACK.get((finding.impact, finding.confidence), "Insight") -def classify_all(findings: List["SlitherFinding"]) -> List[dict]: +def classify_all(findings: List["SlitherFinding"]) -> List[Dict[str, Any]]: """Classify findings, attaching an `immunefi_severity` field, sorted high→low.""" rows = [] for f in findings: diff --git a/cyberai/bench/apps/_server.py b/cyberai/bench/apps/_server.py index 05945313..b4920217 100644 --- a/cyberai/bench/apps/_server.py +++ b/cyberai/bench/apps/_server.py @@ -21,7 +21,7 @@ def _query(path: str) -> Dict[str, str]: class BenchHandler(BaseHTTPRequestHandler): """Routes are supplied by each app as {(method, path): handler}.""" - routes: Dict[tuple, Callable[["BenchHandler"], Any]] = {} + routes: Dict[tuple[str, str], Callable[["BenchHandler"], Any]] = {} def log_message(self, fmt: str, *args: Any) -> None: # keep output quiet pass diff --git a/cyberai/core/exploit_memory.py b/cyberai/core/exploit_memory.py index b44cc2e5..76661351 100644 --- a/cyberai/core/exploit_memory.py +++ b/cyberai/core/exploit_memory.py @@ -63,7 +63,7 @@ def build_record( } -def _cosine(a: Counter, b: Counter) -> float: +def _cosine(a: Counter[str], b: Counter[str]) -> float: common = set(a) & set(b) if not common: return 0.0 diff --git a/docs/architecture/typing-scope.md b/docs/architecture/typing-scope.md index 87d59a25..20aedc0c 100644 --- a/docs/architecture/typing-scope.md +++ b/docs/architecture/typing-scope.md @@ -1,6 +1,6 @@ # Typing scope -`mypy --strict` reads 104 of 172 modules in the package. The other 68 hold 275 +`mypy --strict` reads 107 of 172 modules in the package. The other 65 hold 272 errors and are not checked. The scope is a list of named modules, so a module that passes strictly stays @@ -13,7 +13,7 @@ module that could be declared and is not becomes a failing CI step rather than a quiet omission. Not checked is stronger than it sounds, and the boundary is the reason. Of -the 104 modules in the scope, 22 import a module outside it at module level, +the 107 modules in the scope, 22 import a module outside it at module level, and between them they reach 30 such modules. mypy follows those imports to resolve names and does not report what it finds there: measured on 2026-09-17 by appending an unannotated function to `cyberai/core/config.py`, which was @@ -68,8 +68,28 @@ before the fact, which is what the numbers here are for. A third module went in the same day, `agents/exploit/safety_validator.py`, whose public signature declared two implicit Optionals. Only one counter moved: it imports nothing outside the scope, so it never was a crosser, and -reached fell from 31 to 30. A leaf pays one counter and an entry point pays -both, which is the shape of the rule rather than a number to memorise. +reached fell from 31 to 30. + +Those three moves were once summarised here as a rule -- a leaf pays one +counter, an entry point pays both. `scripts/scope_price.py` projected all 68 +undeclared modules on 2026-09-23 and the summary does not survive it. The +moves fall into eleven classes, not two, and `core/cache.py` is the plainest +refutation: it imports nothing outside the scope, so by that rule it was a +leaf paying one counter, and it moves both. `agents/intel/epss_client.py` is +the only module in the scope that reaches it and reaches nothing else, so +declaring cache.py retires a crosser and a reached module at once. + +What the projection shows instead is two independent sums. A candidate +enters the crosser set if it imports past the edge itself, and it removes +every module whose only crossing was to reach it; `integrations/phantom_grid.py` +removes two, which no rule about leaves permits. It adds its own targets to +the reached set and removes itself if anything already reached it; +`core/orchestrator.py` adds two while joining no crossing of its own. Because +the two sums are independent, the observed pairs run from -2 and -1 up to +1 +and +5, and the largest class is neither: twenty modules move nothing at all, +costing only the errors they carry. The price of a module is therefore read +per module and before the fact, which is what the tool is for and what this +paragraph no longer claims to predict. ## How the set was drawn diff --git a/pyproject.toml b/pyproject.toml index 4792ad0c..6cfa9145 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,6 +136,7 @@ files = [ "cyberai/agents/web3/etherscan.py", "cyberai/agents/web3/foundry_poc.py", "cyberai/agents/web3/halmos_tool.py", + "cyberai/agents/web3/immunefi_severity.py", "cyberai/agents/web3/judge.py", "cyberai/agents/web3/poc_synth.py", "cyberai/agents/web3/property_synth.py", @@ -143,6 +144,7 @@ files = [ "cyberai/bench/__init__.py", "cyberai/bench/agent_engine.py", "cyberai/bench/apps/__init__.py", + "cyberai/bench/apps/_server.py", "cyberai/bench/ctf.py", "cyberai/bench/ctf_loader.py", "cyberai/bench/cve_bench.py", @@ -166,6 +168,7 @@ files = [ "cyberai/core/config.py", "cyberai/core/cost_tracker.py", "cyberai/core/egress_guard.py", + "cyberai/core/exploit_memory.py", "cyberai/core/llm_usage.py", "cyberai/core/model_router.py", "cyberai/core/phase_map.py", diff --git a/scripts/scope_price.py b/scripts/scope_price.py new file mode 100644 index 00000000..a5e6b5da --- /dev/null +++ b/scripts/scope_price.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""What a module costs before it is added to `[tool.mypy] files`. + +Three times on 2026-09-21 the boundary counters moved as modules entered the +scope, and three times the expected numbers in the guard were corrected after +the fact by rerunning it and reading the failure. That works and teaches +nothing: the price was paid first and read afterwards. The prose on the scope +page states the rule -- a leaf pays one counter, an entry point pays both -- +and the rule was written from those three moves rather than measured against +a fourth. + +This reads the price first. For a candidate module it reports the errors the +module itself brings under the declared flags, and the two boundary counters +as they would stand with the module inside the scope. Nothing in the tree is +edited to get there: the scoped run is driven by a config file written to a +temporary directory, so `pyproject.toml` is never touched and an interrupted +run leaves no half-applied scope behind. + +The counters are counted the way the guard counts them, from the import graph +rather than from the checker, because that is the measurement the guard will +compare against. A number produced by a different method would agree with the +guard by luck and diverge without warning. + +A run that produces no verdict line is not a clean module. `mypy` absent from +the environment writes nothing to standard output, and every count taken from +that silence reads as zero errors on a module nobody checked. +""" + +from __future__ import annotations + +import argparse +import ast +import pathlib +import re +import subprocess +import sys +import tempfile +import tomllib + +_ROOT = pathlib.Path(__file__).resolve().parents[1] +_PYPROJECT = _ROOT / "pyproject.toml" +_PACKAGE = _ROOT / "cyberai" + +_ERROR = re.compile(r"^(?P[^:]+\.py):\d+: error") +_VERDICT = re.compile(r"^(Found \d+ error|Success: no issues found)", re.MULTILINE) + + +def settings() -> dict[str, object]: + loaded: dict[str, object] = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8"))["tool"][ + "mypy" + ] + return loaded + + +def declared_scope(config: dict[str, object]) -> set[pathlib.Path]: + declared = config["files"] + if not isinstance(declared, list): + raise TypeError("[tool.mypy] files is not a list; the scope cannot be resolved") + resolved: set[pathlib.Path] = set() + for entry in declared: + path = _ROOT / str(entry) + if path.is_dir(): + resolved.update(path.rglob("*.py")) + else: + resolved.add(path) + return resolved + + +def _resolve(dotted: str) -> pathlib.Path | None: + base = _ROOT / pathlib.Path(dotted.replace(".", "/")) + for candidate in (base.with_suffix(".py"), base / "__init__.py"): + if candidate.exists(): + return candidate + return None + + +def _imported_names(module: pathlib.Path) -> list[str]: + tree = ast.parse(module.read_text(encoding="utf-8")) + names: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.extend(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + package = module.parent + for _ in range(node.level - 1): + package = package.parent + dotted = str(package.relative_to(_ROOT)).replace("/", ".") + if node.module: + dotted = f"{dotted}.{node.module}" + else: + dotted = node.module or "" + names.append(dotted) + names.extend(f"{dotted}.{alias.name}" for alias in node.names) + return names + + +def crossings(declared: set[pathlib.Path]) -> tuple[set[pathlib.Path], set[pathlib.Path]]: + """Modules importing past the edge, and the modules they reach.""" + crossers: set[pathlib.Path] = set() + reached: set[pathlib.Path] = set() + for module in sorted(declared): + for dotted in _imported_names(module): + if not dotted.startswith("cyberai"): + continue + target = _resolve(dotted) + if target is not None and target not in declared: + crossers.add(module) + reached.add(target) + return crossers, reached + + +def _config_text(config: dict[str, object], files: list[str]) -> str: + lines = ["[tool.mypy]"] + for key in ("python_version", "strict", "ignore_missing_imports"): + if key not in config: + continue + value = config[key] + if isinstance(value, bool): + lines.append(f"{key} = {str(value).lower()}") + else: + lines.append(f'{key} = "{value}"') + lines.append("files = [") + lines.extend(f' "{entry}",' for entry in files) + lines.append("]") + return "\n".join(lines) + "\n" + + +def run_scoped(config: dict[str, object], files: list[str]) -> subprocess.CompletedProcess[str]: + """Run the checker over an arbitrary file set without editing the tree.""" + with tempfile.TemporaryDirectory() as tmp: + written = pathlib.Path(tmp) / "mypy.toml" + written.write_text(_config_text(config, files), encoding="utf-8") + return subprocess.run( + [ + sys.executable, + "-m", + "mypy", + "--config-file", + str(written), + "--cache-dir", + str(pathlib.Path(tmp) / "cache"), + ], + cwd=_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def errors_in(output: str, module: pathlib.Path) -> int: + relative = module.relative_to(_ROOT).as_posix() + return sum( + 1 + for line in output.splitlines() + if (match := _ERROR.match(line)) and match.group("module") == relative + ) + + +def price(config: dict[str, object], module: pathlib.Path) -> dict[str, int] | None: + """Errors the module brings and where the two counters land with it inside.""" + declared = [str(entry) for entry in config["files"]] # type: ignore[union-attr] + candidate = module.relative_to(_ROOT).as_posix() + completed = run_scoped(config, [*declared, candidate]) + if not _VERDICT.search(completed.stdout): + return None + scope = declared_scope(config) + before_crossers, before_reached = crossings(scope) + after_crossers, after_reached = crossings(scope | {module}) + return { + "errors": errors_in(completed.stdout, module), + "crossers": len(after_crossers), + "reached": len(after_reached), + "delta_crossers": len(after_crossers) - len(before_crossers), + "delta_reached": len(after_reached) - len(before_reached), + } + + +def outside(config: dict[str, object]) -> list[pathlib.Path]: + return sorted(set(_PACKAGE.rglob("*.py")) - declared_scope(config)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "module", nargs="*", help="paths under cyberai/; default: the whole outside" + ) + args = parser.parse_args() + config = settings() + if args.module: + candidates = [(_ROOT / entry).resolve() for entry in args.module] + else: + candidates = outside(config) + scope = declared_scope(config) + crossers, reached = crossings(scope) + print(f"scope: {len(scope)} modules, {len(crossers)} crossers reaching {len(reached)}") + print(f"{'module':<52} {'errors':>6} {'crossers':>9} {'reached':>8}") + for module in candidates: + if not module.exists(): + print(f"{module.relative_to(_ROOT).as_posix():<52} {'absent':>6}") + return 2 + if module in scope: + print(f"{module.relative_to(_ROOT).as_posix():<52} {'in':>6}") + continue + measured = price(config, module) + if measured is None: + print( + "the checker returned no verdict: this environment was not measured", + file=sys.stderr, + ) + return 2 + name = module.relative_to(_ROOT).as_posix() + print( + f"{name:<52} {measured['errors']:>6} " + f"{measured['crossers']:>4} {measured['delta_crossers']:>+4} " + f"{measured['reached']:>4} {measured['delta_reached']:>+4}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/typing_scope_drift.py b/scripts/typing_scope_drift.py index d428e08e..45e75b37 100644 --- a/scripts/typing_scope_drift.py +++ b/scripts/typing_scope_drift.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 """Report modules that pass `mypy --strict` while sitting outside the scope. -`[tool.mypy] files` names five directories and seventy-two individual modules. +`[tool.mypy] files` names five directories and eighty-five individual modules +(measured 2026-09-23; the count moves as modules are declared). A directory keeps its guarantee open: a module added inside one is checked from the moment it lands. A named module does not. A sibling dropped next to one of -the seventy-two passes strictly, is never checked, and nothing says so. The +the named ones passes strictly, is never checked, and nothing says so. The declared scope then understates what the repository holds, and it understates it more with every module added. diff --git a/tests/architecture/test_the_price_of_a_module_is_measured_not_assumed.py b/tests/architecture/test_the_price_of_a_module_is_measured_not_assumed.py new file mode 100644 index 00000000..8bd862b7 --- /dev/null +++ b/tests/architecture/test_the_price_of_a_module_is_measured_not_assumed.py @@ -0,0 +1,122 @@ +"""The price reported before a module is added must be the price the guard charges. + +Two files count the same crossing. test_typing_scope_boundary_is_declared holds +the numbers the scope page states; scripts/scope_price.py projects where those +numbers land when a candidate module moves inside. Two counters over one import +graph agree today because one was written from the other, and nothing keeps +them agreeing: the day one follows relative imports differently, the price read +before the change and the price charged after it diverge, and the second is the +one that reds. So the agreement is asserted here rather than assumed. + +The projection is asserted through a module already inside the scope. Adding +one changes nothing -- it crosses no edge it did not cross before, and it is +reached by nobody it did not already belong to -- so both counters must hold +still. A pricer that stopped asking whether a target sits outside the scope +would report the whole import graph as crossings and move them. + +The refusal is asserted the same way the drift report's is: a run that emits no +verdict line is not a module that costs nothing. An environment without the +checker writes nothing to standard output, and a price taken from that silence +reads as zero errors on a module nobody read. + +The script is loaded by path. scripts/ resolves today only because of how the +package is installed, and a gate resting on the install mode is a gate about +one machine. +""" + +import importlib.util +import pathlib +import subprocess +import types + +import pytest + +_ROOT = pathlib.Path(__file__).resolve().parents[2] +_SCRIPT = _ROOT / "scripts" / "scope_price.py" +_GUARD = pathlib.Path(__file__).with_name("test_typing_scope_boundary_is_declared.py") + + +def _by_path(name: str, path: pathlib.Path) -> types.ModuleType: + """Load by location, never by import root. + + The sibling guard is a test module, and importing it as tests.architecture + asks the suite to be a package on sys.path. It is not one: the manifest + gate reads that import as a third-party dependency named tests, and the + dependency it would really name is the install mode. + """ + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader, f"no module at {path}" + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pricer() -> types.ModuleType: + return _by_path("scope_price", _SCRIPT) + + +def test_the_pricer_counts_the_edge_the_boundary_guard_counts() -> None: + """One import graph, two readers, one pair of numbers.""" + guard = _by_path("boundary_guard", _GUARD) + module = _pricer() + crossers, reached = module.crossings(module.declared_scope(module.settings())) + assert len(crossers) == guard._EXPECTED_CROSSERS + assert len(reached) == guard._EXPECTED_REACHED + + +def test_a_module_already_inside_the_scope_costs_no_movement() -> None: + """The projection must ask which side of the edge a target sits on.""" + module = _pricer() + scope = module.declared_scope(module.settings()) + inside = _ROOT / "cyberai" / "agents" / "exploit" / "safety_validator.py" + assert inside in scope, "the module this test reasons about left the scope" + before = module.crossings(scope) + after = module.crossings(scope | {inside}) + assert (len(after[0]), len(after[1])) == (len(before[0]), len(before[1])) + + +def test_a_run_without_a_verdict_is_not_a_module_that_costs_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Silence from an absent checker must not be read as an empty error list.""" + module = _pricer() + absent = subprocess.CompletedProcess(["mypy"], 1, stdout="", stderr="No module named mypy") + monkeypatch.setattr(module.subprocess, "run", lambda *a, **k: absent) + priced = module.price(module.settings(), _ROOT / "cyberai" / "core" / "cache.py") + assert priced is None + + +def test_a_verdict_with_no_error_lines_prices_a_module_at_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The refusal above must not swallow the honest clean run as well.""" + module = _pricer() + clean = subprocess.CompletedProcess( + ["mypy"], 0, stdout="Success: no issues found in 105 source files\n", stderr="" + ) + monkeypatch.setattr(module.subprocess, "run", lambda *a, **k: clean) + priced = module.price(module.settings(), _ROOT / "cyberai" / "core" / "cache.py") + assert priced is not None + assert priced["errors"] == 0 + + +def test_the_count_belongs_to_the_module_it_is_charged_to() -> None: + """Two mutants survived the fakes above: neither fake emits an error line. + + A pricer that counted every error line in the run, or matched any annotated + line rather than an error, would charge a candidate for the whole package + and for its notes as well. The output is read here as text, so the parsing + is asserted without paying for a run. + """ + module = _pricer() + output = "\n".join( + [ + "cyberai/core/cache.py:62: error: Missing type parameters [type-arg]", + "cyberai/core/cache.py:70: error: Returning Any [no-any-return]", + "cyberai/core/cache.py:71: note: Consider annotating it", + "cyberai/core/logger.py:14: error: Missing type parameters [type-arg]", + "Found 3 errors in 2 files (checked 105 source files)", + ] + ) + assert module.errors_in(output, _ROOT / "cyberai" / "core" / "cache.py") == 2 + assert module.errors_in(output, _ROOT / "cyberai" / "core" / "logger.py") == 1 diff --git a/tests/architecture/test_typing_scope_boundary_is_declared.py b/tests/architecture/test_typing_scope_boundary_is_declared.py index 1c56a6b2..da8170d8 100644 --- a/tests/architecture/test_typing_scope_boundary_is_declared.py +++ b/tests/architecture/test_typing_scope_boundary_is_declared.py @@ -1,9 +1,10 @@ """The typing scope has an edge, and until now nothing said where. -`[tool.mypy] files` names 95 modules and the run reports `Success` on exactly -95, which reads like a guarantee about those modules and is one only up to -the boundary. Nineteen of them import a module outside the scope at module -level. mypy follows such an import to resolve the name and stays silent about +`[tool.mypy] files` resolves to 107 modules and the run reports `Success` on +exactly 107, which reads like a guarantee about those modules and is one only +up to the boundary. Twenty-two of them import a module outside the scope at +module level. Both numbers were 95 and 19 when this was written and are +restated here on 2026-09-23; the assertions below are what holds them. mypy follows such an import to resolve the name and stays silent about what it finds: appending an unannotated function to `cyberai/core/config.py`, which is outside the scope and imported from inside it, changed nothing about the output on a cold cache. A name crossing the edge is therefore typed by a @@ -37,6 +38,9 @@ # counts and the direction depends on which side of the import it sits. # __main__.py reaches three modules nothing else in the scope reaches: # cli/audit_verify.py, cli/detector_eval.py, cli/scope.py. +# 2026-09-23: immunefi_severity.py, bench/apps/_server.py and +# core/exploit_memory.py were declared, priced at zero on both counters by +# scripts/scope_price.py before the fact and moving neither afterwards. # Third move the same day: agents/exploit/safety_validator.py was declared, # and only reached fell, 31 to 30. It imports nothing outside the scope, so # it never became a crosser; it stopped being reached. A leaf costs one @@ -103,7 +107,7 @@ def _crossings() -> tuple[set[pathlib.Path], set[pathlib.Path]]: def test_the_scope_covers_the_modules_it_declares() -> None: """The premise the rest of this file argues about.""" - assert len(_scope()) == 104 + assert len(_scope()) == 107 assert len(list(_PACKAGE.rglob("*.py"))) == 172