diff --git a/README.md b/README.md index 67c3039..e9a785c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue) ![License](https://img.shields.io/badge/license-Apache_2.0-blue) ![Version](https://img.shields.io/badge/version-v1.7.0-brightgreen) -![Tests](https://img.shields.io/badge/tests-2988%20collected-brightgreen) +![Tests](https://img.shields.io/badge/tests-3001%20collected-brightgreen) ![Mypy](https://img.shields.io/badge/mypy-strict%3A%20107%2F172%20modules-blue) ![LLM](https://img.shields.io/badge/LLM-OpenAI%20%7C%20Anthropic%20%7C%20Ollama-blueviolet) ![Air-Gapped](https://img.shields.io/badge/air--gapped-ready-success) @@ -417,6 +417,7 @@ methodology and the current scorecard. | [docs/benchmarks/cve-bench.md](docs/benchmarks/cve-bench.md) | The external suite, scored 0/3, with the cause | | [docs/architecture/risk-register.md](docs/architecture/risk-register.md) | Twenty-two risks to the project, each with a status and the test that holds it | | [docs/architecture/typing-scope.md](docs/architecture/typing-scope.md) | What the type checker reads, what it does not, and why | +| [docs/architecture/refusal.md](docs/architecture/refusal.md) | Every way the tool says no, and why one reader narrows where another refuses | | [docs/workflows/htb-with-cyberai.md](docs/workflows/htb-with-cyberai.md) | Walkthrough: a lab box end to end | | [docs/workflows/web3-discovery.md](docs/workflows/web3-discovery.md) | Walkthrough: contract discovery to Immunefi export | | [docs/usage/examples.md](docs/usage/examples.md) | Command recipes by task | diff --git a/blog/launch-post-draft.md b/blog/launch-post-draft.md index 22663f6..78dd34a 100644 --- a/blog/launch-post-draft.md +++ b/blog/launch-post-draft.md @@ -13,7 +13,7 @@ not exist yet, it says so. CyberAI is a multi-agent offensive-security platform: eight agents (recon, intel, exploit, report, planner, mcp-scan, redteam, web3) run a typed, audited pipeline over a shared knowledge base. -2988 tests collected under the gated selection run before every commit, with the +3001 tests collected under the gated selection run before every commit, with the slow and smoke tests deselected there and run separately, `mypy --strict` clean over 107 of 172 modules, Apache-2.0. diff --git a/cyberai/utils/backoff.py b/cyberai/utils/backoff.py index 97cb357..6355cf1 100644 --- a/cyberai/utils/backoff.py +++ b/cyberai/utils/backoff.py @@ -33,6 +33,16 @@ def exponential_backoff( max_delay: cap on delay exceptions: exception types that trigger retry + Raises: + ValueError: max_retries below one. Asked for zero attempts, this + helper used to reach its own final raise with nothing + caught and fail with "exceptions must derive from + BaseException" -- a TypeError from the retry helper, + naming neither the call that failed nor the reason. + Zero attempts is a caller's mistake and is answered at + once, before any request goes out. Reading it as one + attempt would invent a choice nobody made. + Usage: result = exponential_backoff( nvd_client.fetch_cve, @@ -41,6 +51,9 @@ def exponential_backoff( exceptions=(httpx.HTTPStatusError,), ) """ + if max_retries < 1: + raise ValueError(f"max_retries must be at least 1, got {max_retries}") + last_exc: Optional[Exception] = None for attempt in range(max_retries): @@ -59,26 +72,15 @@ def exponential_backoff( time.sleep(delay) logger.error(f"[backoff] {fn.__name__} failed after {max_retries} attempts") + if last_exc is None: # pragma: no cover - unreachable while the guard stands + # Unreachable while max_retries >= 1: the loop runs and either + # returns or binds last_exc. The checker cannot see that, and the + # honest way to say so is a branch that reports a broken invariant + # rather than a type: ignore that hides one. Before the guard above + # this line raised None and the caller read "exceptions must derive + # from BaseException" instead of the failure that actually happened. + raise RuntimeError( + f"[backoff] {fn.__name__} ended with no attempt and no failure " + f"(max_retries={max_retries})" + ) raise last_exc - - -class RateLimitError(Exception): - """Raised when API returns 429 Too Many Requests.""" - - pass - - -def nvd_backoff(fn: Callable, *args, **kwargs) -> Any: - """ - NVD-specific backoff: longer delays, respects 6s/request NVD limit. - NVD API 2.0 rate limit: 5 requests per 30s without API key. - """ - return exponential_backoff( - fn, - *args, - max_retries=5, - base_delay=6.0, # NVD recommends 6s between requests - max_delay=120.0, - exceptions=(Exception,), - **kwargs, - ) diff --git a/docs/architecture/refusal.md b/docs/architecture/refusal.md new file mode 100644 index 0000000..8d38153 --- /dev/null +++ b/docs/architecture/refusal.md @@ -0,0 +1,106 @@ +# How CyberAI Refuses + +An offensive tool that proceeds when it should stop is worse than one that +stops when it should proceed: the first sends packets nobody authorised, the +second wastes a minute. This page says where the refusals are, what each one +says out loud, and why two readers of the same environment variable answer +differently on purpose. + +Nothing here is a list kept by hand. `tests/architecture/test_every_refusal_is_on_the_page.py` +walks the package with AST, collects every class that inherits from an +exception, and fails if one of them is missing below -- or if a name below +no longer exists in the tree. + +## What a refusal costs the reader + +A refusal is only useful if somebody learns of it. The tree holds four +shapes, and they differ by who is standing at the screen: + +**Raised and left to travel.** The caller decides what the message becomes. +This is the shape for a stop that must not be swallowed: an unauthorised +egress, a budget already spent. + +**Raised, caught, and turned into one line.** A command-line reader gets a +sentence instead of a traceback. `click.ClickException` is the wrapper, and +the original is chained with `from exc` so the cause survives in a debug run. + +**Raised, caught, and turned into a value.** The run continues with a +substitute. Only one refusal does this, and only when the caller supplied +the substitute in advance. + +**Not raised at all.** The reader narrows to a default instead. This is not +a refusal to act -- it is a refusal to invent a choice nobody made, and it +is the subject of the second half of this page. + +## Every refusal in the tree + +| exception | raised in | what the reader sees | +| --- | --- | --- | +| `EgressViolation` | `core/egress_guard.py` | air-gapped mode was asked for and the endpoint is not local; the message names provider, base URL and the two ways to fix it | +| `BudgetExceeded` | `core/cost_tracker.py` | the spend cap set by `CYBERAI_MAX_COST_USD` was reached; the call is not made | +| `ToolInputBlocked` | `core/safety.py` | a tool argument failed validation before the tool ran | +| `SealedEnvError` | `core/sandbox/proc.py` | a subprocess was asked to start with an environment the sandbox will not seal | +| `AgentIterationLimitError` | `core/base_agent.py` | an agent loop hit its iteration ceiling rather than running unbounded | +| `InjectionBlocked` | `core/security/guard.py` | under the `deny` policy, a prompt-injection verdict stops the call before any provider is contacted | +| `CorpusError` | `core/security/eval_corpus.py` | the evaluation corpus could not be read; surfaces as one CLI line | +| `RecordMismatch` | `core/security/llm_classifier.py` | a replay recording does not match the corpus it is replayed against; surfaces as one CLI line | +| `AgentTimeoutError` | `core/timeout.py` | nothing, when a fallback was supplied; otherwise it travels | + +This table is generated from the tree rather than written once, and the +first thing it found was a refusal that did not exist. `RateLimitError` sat +in `utils/backoff.py` declaring that it was raised on 429, with no code +raising it, no code catching it and no code importing it. A refusal nobody +raises is indistinguishable, from the outside, from one that never fires. +It was removed on 2026-09-23; the 429 the class described is real and is +already answered, by `httpx.HTTPStatusError` in both NVD request paths. + +`InjectionBlocked` is caught once on its way out, in `core/llm_client.py`, +and re-raised. The catch exists to write the verdict to the audit trail +before the exception propagates: a blocked call is the one most worth having +in the record, so the refusal is documented first and delivered second. + +## Refusing to invent a choice + +Six readers translate environment variables into configuration. None of them +raises, and that is deliberate: `from_env` has no raise in it at all, nine +call sites reach it, and one of those is inside an MCP tool where an +exception is not a message anybody reads. Garbage in a variable must not +abort a scan at startup. + +What they do instead is narrow. A value outside the declared set is read as +nobody having chosen, so the default stands. + +| reader | reads | a value it does not recognise | +| --- | --- | --- | +| `_env_bool` | feature flags that default to off | off, which is the default anyway | +| `_env_guard_bool` | flags whose default protects the run | the default, which stays on | +| `_env_int` | whole numbers with a default | the default | +| `_env_optional_int` | whole numbers with no default | unset, which the caller can tell from a real choice | +| `_env_float` | decimals with a default | the default | +| `_env_provider` | the LLM provider name | the default provider | + +**Two boolean readers, because the words for "no" are not the words for +"nobody chose".** `_env_bool` asks whether a value is one of the words for +yes, so everything else reads as no. For the flags that default to off that +is the same answer as the default and costs nothing. For a flag that +defaults to on it is the difference between a guard and no guard: measured +on 2026-09-19, `CYBERAI_STRICT_SCOPE=` -- the empty string a shell leaves +behind for an unset variable in a `.env` file -- turned the scope refusal +off, and so did `CYBERAI_STRICT_SCOPE=nope`. `_env_guard_bool` holds the +named list in both directions and leaves the default standing otherwise. + +**The environment narrows, the flag refuses.** `CYBERAI_LLM_PROVIDER=gemini` +leaves the default provider in force and the run proceeds; `--provider +gemini` is refused by the parser, which lists the names that exist. The +asymmetry is not an oversight. A stale variable is read by nobody in +particular, possibly long after whoever set it moved on, and aborting a scan +there helps no one. A flag is typed by a person who is at the terminal right +now and will read the reply. `cyberai status` prints the provider actually +in force, which is where an unrecognised variable gets said out loud. + +## What this page does not cover + +Refusals that belong to a library rather than to this package -- a `click` +parser error, a `requests` timeout, an nmap exit code -- are not listed. +They are real and they reach the user, but a page that tried to enumerate +them would describe somebody else's tree and go stale on their schedule. diff --git a/tests/architecture/test_every_refusal_is_on_the_page.py b/tests/architecture/test_every_refusal_is_on_the_page.py new file mode 100644 index 0000000..39d2d29 --- /dev/null +++ b/tests/architecture/test_every_refusal_is_on_the_page.py @@ -0,0 +1,186 @@ +"""Every exception this package defines must be explained on the refusal page. + +A refusal is a contract with whoever is at the screen. The tree holds ten of +them, spread across seven modules, and until this file existed not one was +named in docs/ or in the README -- ten ways the product says no, documented +nowhere. Prose does not rot loudly: a new exception lands in a module, the +page goes on listing the old set, and nothing disagrees out loud. + +Both directions, for the reason the manifest guard states: a refusal the page +omits is a mechanism no reader can find, and a refusal the page names that +the tree no longer defines is the same rot left behind by a rename. + +The set is collected structurally, not by suffix. A rule keyed on names +ending in Error or Violation reads the current tree correctly and would go +blind the day somebody writes `class Throttle(RuntimeError)` -- which is +exactly the edit this guard exists to catch. Inheritance is resolved through +locally defined classes as well, so a subclass of one of ours counts. + +There is one collector, called twice: once over the package and once over a +sample where the structural rule and the name rule disagree. An earlier +revision had two copies, and mutation showed what that cost -- swapping the +package copy for a name test left every assertion green, because today's +tree happens to agree with both rules. A guard cannot hold a distinction its +own input never exercises. + +The environment readers are checked the same way and for the same reason. +They are the other half of the page: six functions that refuse to invent a +choice rather than refusing to act, and the distinction between them is the +page's subject. A seventh reader added without a row would leave the page +describing a narrowing rule that no longer covers every variable. + +What is not pinned is the wording. A test that pinned sentences would fail on +every edit to a paragraph and teach the reviewer to regenerate prose without +reading it. +""" + +import ast +import pathlib +import re + +_ROOT = pathlib.Path(__file__).resolve().parents[2] +_PACKAGE = _ROOT / "cyberai" +_PAGE = _ROOT / "docs" / "architecture" / "refusal.md" + +# Exception roots from the standard library. A class reaching any of these, +# directly or through one of ours, is a refusal. +_BUILTIN_ROOTS = frozenset( + { + "Exception", + "BaseException", + "RuntimeError", + "ValueError", + "TypeError", + "KeyError", + "OSError", + "LookupError", + "ArithmeticError", + } +) + +# The rule this guard is not allowed to be. Kept as a constant so the control +# below compares against it by name rather than by a second literal. +_NAME_RULE = ("Error", "Violation", "Exceeded", "Blocked", "Mismatch") + +# The first cell of a markdown row when it is a single backticked token. +_ROW = re.compile(r"^\|\s*`([^`|]+)`\s*\|", re.MULTILINE) + + +def _is_exception(name: str, defined: dict[str, list[str]], seen: set[str]) -> bool: + if name in _BUILTIN_ROOTS: + return True + if name in seen or name not in defined: + return False + seen.add(name) + return any(_is_exception(base, defined, seen) for base in defined[name]) + + +def _refusals_in(source: str) -> set[str]: + """Every class in this source that reaches an exception root by inheritance.""" + defined: dict[str, list[str]] = {} + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ClassDef): + defined[node.name] = [ast.unparse(base) for base in node.bases] + return { + name + for name, bases in defined.items() + if any(_is_exception(base, defined, set()) for base in bases) + } + + +def _tree_refusals() -> set[str]: + """Every refusal the package defines, read from the tree on disk.""" + joined = "\n".join(path.read_text(encoding="utf-8") for path in sorted(_PACKAGE.rglob("*.py"))) + found = _refusals_in(joined) + assert found, "no exceptions found in the package -- the walker broke" + return found + + +def _env_readers() -> set[str]: + """Every environment reader in the config module.""" + source = (_PACKAGE / "core" / "config.py").read_text(encoding="utf-8") + found = { + node.name + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.FunctionDef) and node.name.startswith("_env_") + } + assert found, "no environment readers found -- the walker broke" + return found + + +def _section(heading: str) -> str: + """The body under one heading, up to the next one at any level.""" + lines = _PAGE.read_text(encoding="utf-8").splitlines() + start = next(i for i, line in enumerate(lines) if line.strip() == heading) + end = next( + (i for i in range(start + 1, len(lines)) if lines[i].startswith("#")), + len(lines), + ) + return "\n".join(lines[start:end]) + + +def _named_in(heading: str) -> set[str]: + named = set(_ROW.findall(_section(heading))) + assert named, f"the section {heading!r} holds no table" + return named + + +def test_every_refusal_in_the_tree_is_explained_on_the_page() -> None: + missing = _tree_refusals() - _named_in("## Every refusal in the tree") + assert not missing, f"the tree raises what the page never explains: {sorted(missing)}" + + +def test_the_page_explains_no_refusal_the_tree_dropped() -> None: + stale = _named_in("## Every refusal in the tree") - _tree_refusals() + assert not stale, f"the page explains what the tree no longer defines: {sorted(stale)}" + + +def test_every_environment_reader_is_explained_on_the_page() -> None: + missing = _env_readers() - _named_in("## Refusing to invent a choice") + assert not missing, f"readers the page never explains: {sorted(missing)}" + + +def test_the_page_explains_no_reader_that_was_renamed() -> None: + stale = _named_in("## Refusing to invent a choice") - _env_readers() + assert not stale, f"the page explains readers that no longer exist: {sorted(stale)}" + + +def test_the_walker_reads_inheritance_and_not_the_name() -> None: + """Control: on input where the two rules disagree, this collector is right. + + Every exception in the tree today ends in one of the five words above, so + a guard keyed on suffixes passes every assertion here -- mutation + confirmed it on 2026-09-23 by swapping the collector for a name test and + watching the suite stay green. The rule is therefore asserted on input the + package does not contain: a refusal named for what it is rather than for + what it inherits, two generations of subclass, and a helper whose name + ends in Error without being one. + + Three generations rather than two, measured on 2026-09-23. A collector + that checked only the direct bases of a base still answered correctly at + depth two -- asking about Deeper reaches Throttle, and Throttle's own + bases hold RuntimeError -- so the recursion was not exercised by the + shorter sample and a mutant that removed it survived. Depth three is the + first input where a flat check and a recursive one disagree. + + This exercises the same function the four tests above call. A control + running a second implementation would prove nothing about the first. + """ + sample = ( + "class Throttle(RuntimeError):\n" + " pass\n" + "class Deeper(Throttle):\n" + " pass\n" + "class Deepest(Deeper):\n" + " pass\n" + "class ReportsError:\n" + " pass\n" + ) + found = _refusals_in(sample) + assert found == {"Throttle", "Deeper", "Deepest"}, found + + by_name = { + n for n in ("Throttle", "Deeper", "Deepest", "ReportsError") if n.endswith(_NAME_RULE) + } + assert by_name == {"ReportsError"} + assert found != by_name, "the two rules agree on this input -- the control is dead" diff --git a/tests/unit/test_backoff_retries_and_gives_up.py b/tests/unit/test_backoff_retries_and_gives_up.py new file mode 100644 index 0000000..c39d334 --- /dev/null +++ b/tests/unit/test_backoff_retries_and_gives_up.py @@ -0,0 +1,146 @@ +"""The retry helper must retry, must stop, and must re-raise what it caught. + +cyberai/agents/intel/nvd_client.py calls exponential_backoff on every NVD +request, and until this file existed the helper had no test at all. Three +properties matter to the caller and none of them was held: that a call +which succeeds late is not reported as a failure, that a call which never +succeeds stops after max_retries rather than looping, and that the +exception the caller finally sees is the last real one rather than a +substitute. + +Sleeping is patched out. A test that waited for the real delays would take +fourteen seconds at the helper's own defaults, and a slow test is a test +that gets marked slow and stops running. + +The exceptions tuple is exercised with a type outside it. The helper +declares which failures are worth retrying, and a helper that retried +everything would pass the first two tests here while turning a typo in a +URL into five identical attempts. +""" + +import pytest + +from cyberai.utils.backoff import exponential_backoff + + +class _Boom(Exception): + pass + + +class _Other(Exception): + pass + + +def test_a_call_that_succeeds_on_the_last_attempt_is_not_a_failure(monkeypatch): + monkeypatch.setattr("time.sleep", lambda _: None) + calls = {"n": 0} + + def flaky(): + calls["n"] += 1 + if calls["n"] < 3: + raise _Boom("not yet") + return "ok" + + assert exponential_backoff(flaky, max_retries=3, exceptions=(_Boom,)) == "ok" + assert calls["n"] == 3 + + +def test_the_helper_stops_after_max_retries(monkeypatch): + monkeypatch.setattr("time.sleep", lambda _: None) + calls = {"n": 0} + + def always(): + calls["n"] += 1 + raise _Boom("never") + + with pytest.raises(_Boom): + exponential_backoff(always, max_retries=4, exceptions=(_Boom,)) + assert calls["n"] == 4, "the ceiling is the number of attempts, not of sleeps" + + +def test_the_exception_the_caller_sees_is_the_last_one_raised(monkeypatch): + """Not the first, and not a wrapper: the caller reads the final cause.""" + monkeypatch.setattr("time.sleep", lambda _: None) + calls = {"n": 0} + + def numbered(): + calls["n"] += 1 + raise _Boom(f"attempt-{calls['n']}") + + with pytest.raises(_Boom, match="attempt-3"): + exponential_backoff(numbered, max_retries=3, exceptions=(_Boom,)) + + +def test_a_failure_outside_the_declared_set_is_not_retried(monkeypatch): + """Control: a helper that retried everything passes the tests above.""" + monkeypatch.setattr("time.sleep", lambda _: None) + calls = {"n": 0} + + def wrong_kind(): + calls["n"] += 1 + raise _Other("not for retrying") + + with pytest.raises(_Other): + exponential_backoff(wrong_kind, max_retries=5, exceptions=(_Boom,)) + assert calls["n"] == 1, "a failure the caller did not name must travel at once" + + +def test_zero_attempts_is_refused_before_anything_is_called(monkeypatch): + """The helper answers the caller rather than failing inside itself. + + Measured on 2026-09-23: max_retries=0 walked past an empty loop to the + final raise with nothing caught, and Python answered `raise None` with + "exceptions must derive from BaseException" -- a TypeError naming + neither the call nor the reason, logged one line after a claim that the + call had failed after zero attempts. mypy had been reporting the same + line as [misc] the whole time; the module sits outside [tool.mypy] files, + so the checker saw it and the gate did not. + """ + monkeypatch.setattr("time.sleep", lambda _: None) + calls = {"n": 0} + + def never_called(): + calls["n"] += 1 + return "unreachable" + + with pytest.raises(ValueError, match="at least 1"): + exponential_backoff(never_called, max_retries=0, exceptions=(_Boom,)) + assert calls["n"] == 0, "the refusal comes before the first call, not after" + + +def test_a_negative_ceiling_is_refused_too(monkeypatch): + """range() swallows a negative the same way it swallows zero.""" + monkeypatch.setattr("time.sleep", lambda _: None) + with pytest.raises(ValueError, match="got -1"): + exponential_backoff(lambda: "x", max_retries=-1, exceptions=(_Boom,)) + + +def test_one_attempt_is_allowed_and_calls_once(monkeypatch): + """Control: the guard must refuse zero without refusing the smallest run. + + A guard written as `max_retries < 2`, or as a truthiness check on a + value that is then decremented, passes both tests above. + """ + monkeypatch.setattr("time.sleep", lambda _: None) + calls = {"n": 0} + + def once(): + calls["n"] += 1 + raise _Boom("only attempt") + + with pytest.raises(_Boom, match="only attempt"): + exponential_backoff(once, max_retries=1, exceptions=(_Boom,)) + assert calls["n"] == 1 + + +def test_no_sleep_happens_after_the_last_attempt(monkeypatch): + """A delay nobody waits through is a delay the caller pays for nothing.""" + slept: list[float] = [] + monkeypatch.setattr("time.sleep", lambda d: slept.append(d)) + + def always(): + raise _Boom("never") + + with pytest.raises(_Boom): + exponential_backoff(always, max_retries=3, exceptions=(_Boom,)) + assert len(slept) == 2, slept