diff --git a/CHANGELOG.md b/CHANGELOG.md index 085276c2..36360266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2950,6 +2950,18 @@ step; each entry says what to use instead. ### Fixed +- **After a failed turn, a call that returned keeps its result on ADK too.** + ADK stores a step's parallel results together, after the last, so a turn + that failed in between (its one `ERROR`: a sibling call's permission check + that raised, a store that refused the write) left a call that had returned + without a stored result, and the next turn answered it "Interrupted before + it finished", though it had run; a delegation's run ended `completed` + while its coordinator read the interruption. ADK now keeps the failed + turn's record of the calls that returned (`OpenCalls.failed`), as Deep + Agents does: the next turn answers each with its result and the state it + changed (`left_open`), any other as interrupted, and forgets the record + once those answers are stored, so it answers no later call of the same id. + - **A timed-out `execute_python` stops the code and what it started.** The timeout killed only the process the executor started (the shell, or bubblewrap's monitor), so the code ran on: on Linux the sandbox's init, diff --git a/CLAUDE.md b/CLAUDE.md index e395f891..23986d3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,7 +84,7 @@ agentic-cli/ │ │ │ └── rules.py, store.py, prompt.py (build_request(run=): a sub-agent's prompt opens with its run; is_permission_request: by its perm- id) │ │ ├── agents/ # Sub-agents: runs.py: the RUN_* statuses (a working run's: RUN_QUEUED, RUN_RUNNING, RUN_INPUT_REQUIRED, RUN_WORKING_STATUSES; a record says running from its start), RunRecord, AgentRuns (a manager's runs by conversation: run_for names - and its run #1 per (conversation, parent run, call id), a new one once it ended; number_from: past the instances a conversation's store keeps, a count never lowered; observe records AGENT_RUN under the turn it is reported in; interrupt_open; a record's usage: what the run used, by model, from its end event or the ledger), run_title, result_head, run_instance and run_number (the one reader of a run id: researcher-2#1's instance and number), run_label (how a dialog names a run: researcher-2 (researcher, run 1)), SUBAGENT_ERROR_PREFIX (ADK's old words for a failed delegation, kept for callers), show_as_delegation(result, envelope) (a delegation's TOOL_RESULT shown alike on both backends: the result's first line, or Failed: ; metadata["result"] the envelope); a record's brief, started_by, mode and cancelled_by; delegate.py: Delegations (delegate(DelegationRequest) → Delegated: the one place a run starts and ends, on both backends: the brief read against the caller's own sub-agents, the run named (a turn's first run past the instances its conversation's store keeps: stored=, asked once per conversation per turn, again after a failed or stopped ask), its AGENT_RUN start and end observed and reported, the backend's hook (RunHook: AgentRun → RunAnswer) run in a task of its own with the run active and no tool call, the end by the answer (completed, partial for a cut or the step limit, refused, failed for an error or a raise, with one recoverable ERROR failed: , or RunFailed's own words, cancelled by its control with what the hook had), interrupted on its turn's cancel, which wins; the envelope as the result; each run under its RunControl, the type's limits over the policy's, kept while it works: control(run), cancel(run_id, by=, error=), the run alone; a call past max_runs_per_turn the error result RUNS_PER_TURN_ERROR, no run; each run in a working slot of its conversation's SlotPool, slots(conversation): policy.max_active slots, made on first use and kept, the root taking none: taken right before its hook and its time limit, queued then running while it waits for one, given back at its end, straight back to a caller that waits on it alone; a caller that delegates lends its slot while it waits for its children), DelegationRequest, AgentRun (control), RunAnswer, RunFailed, started_by, waiting_on_user (a sub-agent run waiting on the user: input_required, its slot lent, then running; nested waits once; request_user_input and the permission engine); slots.py: SlotPool (acquire(priority): a free slot, else a future in a heap by priority, RETURNING before NEW, then arrival; release; lent(slot): the slot lent while its run waits, nested lends once, taken back at RETURNING after, at most one take-back waiting for a slot and none once it is held (a wait that starts while one is queued takes it over, and one handed the slot withdraws any queued since), nothing taken back after a cancel; hand_back(slot, to): a child's slot to its parent waiting on it alone; free, waiting; a cancelled waiter leaves the heap, one handed a slot as it is cancelled hands it on; not asyncio.Semaphore), Slot (held, lent), RETURNING, NEW; control.py: RunControl (one run's: cancel(by=, error=) its task, the first cancel winning, own_cancel for the hook that takes it and returns, timing: the time limit while the hook runs; model_call/at_step_limit/final_call: the step limit, STEP_LIMIT_NOTE on the last allowed call; used: the token limit, an answer past it that makes calls the run's last, cancelled by it, one that makes none its result (calls=); model_call once final stays final; a cancel leaves a task its turn is cancelling alone; steer/take_steering), STEP_LIMIT_NOTE; outcome.py: a sub-agent's refused or cut answer (outcome_of, the refusal notice as Deep Agents 0.7.34 words it, cut_off, cut_off_warning, OPENAI_REFUSAL_PREFIX: how ADK's OpenAI adapters word a refusal); policy.py: SubAgentPolicy (max_active, max_runs_per_turn, limits) and AgentLimits (max_model_calls, timeout, max_tokens; over: a type's limits over the policy's, field by field, None taking the base's), DEFAULT_LIMITS (100 model calls), frozen and checked as they are made (a ValueError naming the field), as_policy (the one reading of a subagents= argument: a TypeError for anything but a policy or None); brief.py: BRIEF_FIELDS, Brief, BriefError, read_brief (a task call's agent type and brief, against the types its caller may start: the error texts), task_message (a run's first message, a section per filled field; an inheriting type's ends with INHERITED_CONTEXT_NOTE), task_parameters and task_description (task's schema and description, the same for both backends; limit_sentences: the limits it states, "At most sub-agents work at once; more calls in one response wait for a free slot. At most runs per turn."); envelope.py: envelope (a delegation run's result: success only for completed, a one-line error, cancelled_by for a cancelled run), read_envelope (a dict or its JSON text, else None; never raises), the limit and interrupted error texts │ │ ├── adk/ # ADK orchestrator -│ │ │ ├── manager.py # GoogleADKWorkflowManager (the agents built from configs or, adk_config_path, by ADK from a native YAML (library-only: no BaseCLIApp starts one; loaded once per manager at the first _validate_agent_graph (_native_root: the settings' keys exported first; a retry, a model switch and a start after cleanup build on the same tree, whose toolsets the closed runner left reusable), its agents' models resolved leniently in _resolve_extra_models, the base's hook, its registered tools' services detected; configs passed with it ignored, adk_native_configs_ignored): the only difference; _do_initialize builds the app from the tree alone, _build_app → ADKBuild, through the hooks _plugin_chain (the whole chain, top to bottom, its head _init_plugins), _compaction_config (None for a root without a budget), _context_cache_config, _agent_model_refs; the event processor built from the build; a config-built agent's prompt as an instruction provider, sent as written: no `{name}` state injection; _compact_turn: ADK's compaction once, in the base's compact turn; supports_compact False once built for a root without a budget (_no_compact_support says why); the job resume hooks: _conversation_exists (the session read without its events, num_recent_events=0), _deliver_job_notice (the notice, for every app); _run_config: the agents' synchronous tools run in ADK's tool thread pool, 8 workers; _run_and_stream: a run that fails yields the turn's one ERROR, last, none after an ERROR already yielded; the agents: _create_agents builds each config's agent on its own (_agents, by name; no ADK sub_agents), a delegating one with a TaskTool (its own tool named task: AgentGraphError); a runner per sub-agent type (_agent_runners: NoAfterTurnCompactionRunner on App(name=child_app(app_name), root_agent=the type's agent, plugins=_agent_plugin_chain(agent), the app's context cache, no compaction) over the session service, closed with the main runner by _detach_backend, rebuilt by every initialization); _run_agent, the base's hook: the run's child session (_child_session: run.state, an inheriting run's copied events; never one the store keeps, whose instance numbering goes past: ADK's create_session refuses it, and the run fails), _stored_instances (the base's hook: the highest instance number of each type the conversation's child sessions keep), the type's runner on the task message with an abort signal set before it is closed, a run-mode ADKEventProcessor, the raw log's records reported as they come (_report_raw_log); a model's failure raised as the provider's (_plugin_cause), an overflow RunFailed, a reply that is no answer RunAnswer(error=), the shared keys it changed read back) +│ │ │ ├── manager.py # GoogleADKWorkflowManager (the agents built from configs or, adk_config_path, by ADK from a native YAML (library-only: no BaseCLIApp starts one; loaded once per manager at the first _validate_agent_graph (_native_root: the settings' keys exported first; a retry, a model switch and a start after cleanup build on the same tree, whose toolsets the closed runner left reusable), its agents' models resolved leniently in _resolve_extra_models, the base's hook, its registered tools' services detected; configs passed with it ignored, adk_native_configs_ignored): the only difference; _do_initialize builds the app from the tree alone, _build_app → ADKBuild, through the hooks _plugin_chain (the whole chain, top to bottom, its head _init_plugins), _compaction_config (None for a root without a budget), _context_cache_config, _agent_model_refs; the event processor built from the build; a config-built agent's prompt as an instruction provider, sent as written: no `{name}` state injection; _compact_turn: ADK's compaction once, in the base's compact turn; supports_compact False once built for a root without a budget (_no_compact_support says why); the job resume hooks: _conversation_exists (the session read without its events, num_recent_events=0), _deliver_job_notice (the notice, for every app); _run_config: the agents' synchronous tools run in ADK's tool thread pool, 8 workers; _run_and_stream: a run that fails yields the turn's one ERROR, last, none after an ERROR already yielded, and keeps the record of its calls that returned for the next turn (OpenCalls.failed; _answer_left_open forgets it once its answers are stored); the agents: _create_agents builds each config's agent on its own (_agents, by name; no ADK sub_agents), a delegating one with a TaskTool (its own tool named task: AgentGraphError); a runner per sub-agent type (_agent_runners: NoAfterTurnCompactionRunner on App(name=child_app(app_name), root_agent=the type's agent, plugins=_agent_plugin_chain(agent), the app's context cache, no compaction) over the session service, closed with the main runner by _detach_backend, rebuilt by every initialization); _run_agent, the base's hook: the run's child session (_child_session: run.state, an inheriting run's copied events; never one the store keeps, whose instance numbering goes past: ADK's create_session refuses it, and the run fails), _stored_instances (the base's hook: the highest instance number of each type the conversation's child sessions keep), the type's runner on the task message with an abort signal set before it is closed, a run-mode ADKEventProcessor, the raw log's records reported as they come (_report_raw_log); a model's failure raised as the provider's (_plugin_cause), an overflow RunFailed, a reply that is no answer RunAnswer(error=), the shared keys it changed read back) │ │ │ ├── build.py # ADKBuild (root, reply_agents, plugins, compaction, cache, models); load_native_root (from_config at validation; what ADK cannot load, wrapped as AgentGraphError naming the file, ADK's error the cause; a root that is not an LlmAgent, a name used twice in the tree and a task-mode agent refused, AgentGraphError naming it; ensure_direct_anthropic); what the run reads of any tree: reply_agents (the root and the chat-mode agents it can transfer to), own_after_tool_agents (left out by the cap) │ │ │ ├── task_tool.py # TaskTool: the framework's task on ADK (a BaseTool; its declaration task_parameters' JSON schema, parameters_json_schema; its description task_description; bound to the task declaration, an exact TaskTool only): a call → Delegations.delegate, the run's shared-key changes written to the caller's state, the envelope returned as JSON text (ADK's OpenAI Responses adapter sends only a dict's result); SHARED_KEYS (plan, tasks, the outside-content mark); inherited_events (an inheriting run's history: the session's root-visible events, copied) │ │ │ ├── compaction.py # AgentCliSummarizer (main agent only; summary_range, render_events; its usage normalized, timed and recorded in the conversation's ledger, ledgers, as soon as the call answers: an empty summary's too; on_written: a summary written drops the conversation's memory index snapshot), MainAgentPlugin (the running agent, first in every chain), NoAfterTurnCompactionRunner, compact_session @@ -97,7 +97,7 @@ agentic-cli/ │ │ │ ├── openai_compatible.py # OpenAICompatibleLlm (the configured providers: openrouter and openai_compatible kinds; reasoning text; OpenRouter's reasoning_details kept on the thought part (thought_signature, with the model that wrote them) and sent back on that turn's assistant message to that model only; a Chat Completions refusal, streamed or not, as the text ADK's Responses adapter makes of one, "OpenAI refusal: "; no field sent as null; OpenRouter's cache writes, which ADK's usage mapping drops, kept on the usage as cache_creation_input_tokens) │ │ │ ├── anthropic_llm.py # DirectAnthropicLlm: bare claude-* strings → direct API (registered once per process, when a native YAML app is loaded: ensure_direct_anthropic) │ │ │ ├── permission_plugin.py # PermissionPlugin (gates tool calls; marks the conversation when an allowed call brings outside content, written to the session state at the model calls of the agents that reply); tool_definition: a tool object's registry identity, the gate's and the tool result cap's -│ │ │ ├── interrupted_calls_plugin.py # InterruptedCallsPlugin: a cancelled turn's seals worded (finished results kept, as the cap cut them), each seal answered as the active run's (a sub-agent run's runner seals its child session in the run's task), each running call marked by its invocation and id; unsealed: the calls ADK's seals missed (they pair by id alone: a native YAML app's branches), answered as the turn stops, from the record; left_open: a crash's calls answered at the next turn (a result pairs with a call on its own branch only; a job's state only for the root's own call, a sub-agent's run being numbered no more), and a stop's whose answers could not be written, as the user's; returned: a finished call's result from the record, as ADK stores it and the model reads it +│ │ │ ├── interrupted_calls_plugin.py # InterruptedCallsPlugin: a cancelled turn's seals worded (finished results kept, as the cap cut them), each seal answered as the active run's (a sub-agent run's runner seals its child session in the run's task), each running call marked by its invocation and id; unsealed: the calls ADK's seals missed (they pair by id alone: a native YAML app's branches), answered as the turn stops, from the record; left_open: a crash's calls answered at the next turn (a result pairs with a call on its own branch only; a job's state only for the root's own call, a sub-agent's run being numbered no more), a failed turn's (the root's that returned with their results, from the record it kept), and a stop's whose answers could not be written, as the user's │ │ │ ├── task_progress_plugin.py # TaskProgressPlugin: deprecated and unused (DeprecationWarning; the event processor reports task progress), removed in 0.8.0 │ │ │ ├── tool_error_plugin.py # Tool exception → error result (turn survives); a call to an unknown tool name → ADK's own text (is_unknown_tool: ADK's stand-in, which the gate lets through), recorded in OpenCalls since ADK skips its after-tool callbacks, so a cancelled turn keeps it │ │ │ ├── tool_result_cap_plugin.py # ToolResultCapPlugin: every tool result through cap_tool_result (after InterruptedCallsPlugin), but an agent's with after-tool callbacks of its own (skip_agents), which ADK skips once a plugin answered @@ -265,7 +265,7 @@ Workflow: - **HITL callback is context-local**: `set_input_callback()` stores into a per-manager `ContextVar`, so a second consumer installing its callback cannot capture a running turn's prompt, and one consumer's `clear_input_callback()` cannot unregister another's. `MessageProcessor` cancels and awaits its consumer task **before** clearing the callback, so no tool is left asking a question nobody owns. And the callback answers only inside its turn: a task a turn started (`asyncio.create_task` copies the context, the callback with it) that asks after the turn ended gets `UserInputUnavailable` (`request_user_input`; the turn's span, `workflow/sessions.py::open_turn_scope`/`close_turn_scope`, opened and closed by `_workflow_context`, which the copy carries), as with no approver: a question goes unanswered and a permission ASK is denied by the engine's no-approver rule. - **Code that may run on the host is approved per call**: CPython has no in-process sandbox, so `execute_python` runs inside the OS sandbox or not at all by default (`os_sandbox_strict` true: no backend, a failed wrap, or a sandbox tool that fails as it starts (the runner's first stderr line, `_STARTED_MARKER`, never came) is a refusal that keeps its cause and ends with `executor.how_to_run`: sandbox-exec, bwrap and Ubuntu 24.04's user namespaces, none on Windows, and the opt-out where it lives, `exec_policy.sandbox_opt_out`: `~/.{app_name}/settings.json` or the env variable, since the project file and `/settings` cannot set it), and the module set is chosen from the path a call takes (`executor.py::_choose_run`, then `validate_code(allowed_modules=)`). The engine never imports `tools/`: `BaseWorkflowManager` passes `PermissionEngine(uncontained=tools.exec_policy.uncontained_exec_capabilities)`, called with the settings at each check (a settings change applies at once). For a capability it names, `engine.unstorable_reason` is `UNCONTAINED` (the one decision, shared with the prompt: only "Allow once" and "Deny", with its note), and `_evaluate` skips allow rules from grants (`SESSION`, and `PROJECT`, the user's `project_grants.json`), so a grant from when the code ran contained approves nothing; a `USER` rule still applies. A prompt for `*.exec` shows all of the code (the dialog body scrolls). - **No harness-level turn replay**: ADK persists a turn's input during invocation setup, and Deep Agents' checkpointer at the turn's first step, so the CLI never re-invokes an event source. Retries belong to the provider client (`HttpRetryOptions`). `MessageProcessor` returns a typed `TurnResult`. A turn that fails yields one `ERROR` and returns, on both backends (the base `process()`'s contract; `WorkflowEvent.turn_failed`: the error's text, `rate_limited`, `error_type`); an `ERROR` the turn already yielded that is not recoverable is its one; a cancel still raises, and a turn that cannot start (initialization, admission, or its conversation cannot be read) raises before any event. The CLI says a rate limit from the event as it does from a manager that raises. -- **Interrupted calls**: Ctrl+C stops the whole turn (loop, model calls, tools, sub-agents), and every call the model made gets exactly one result (`workflow/interrupts.py`, the texts and `OpenCalls`, each manager's record, kept by conversation and by the run that made each call (the root's own, or a sub-agent's: a provider that numbers its call ids per agent gives a sub-agent's call its parent's delegation's id): a call's result as it finished, a call `asking` at its permission prompt, both gates marking it; a conversation whose turn the user cancelled): a finished call keeps its result, a running one or a delegation is `INTERRUPTED_BY_USER`, one at its prompt `NOT_RUN`, one whose job was submitted the job's state (`job_started`); a cancel with no call open leaves `MARKER` as the main agent's reply; a call nothing answered (a crash, an old session) is `INTERRUPTED`. On ADK the turn runs with `abort_signal`, set by `live_events.merged(on_stop=)` before the runner is cancelled, so ADK seals the invocation in its own branches; `merged` waits for its source to close, a repeated cancel too (up to `STOP_GRACE`), and `process()`/`resume_with_job_result()` close their inner generators with `aclosing`, so a run never outlives its turn. `adk/interrupted_calls_plugin.py::InterruptedCallsPlugin` (before `ToolResultCapPlugin`, every app) sets the active tool call around each call (`before_tool_callback`, so a question marks it `asking`), records results with their `state_delta`/`artifact_delta` in `after_tool_callback` (not a long-running starter's `None`: its job answers it), keeps the cap's cut of one (`ToolResultCapPlugin.on_cut`, so a seal sends that cut and its full output is saved once) and rewrites the seals (`error_code` `INVOCATION_ABORTED`, contract-tested) in `on_event_callback`, carrying those deltas, each seal answered from the record of the active run (a sub-agent run's runner stops, and seals its child session, in the run's own task). ADK's seals pair a result with a call by id alone, so a call whose id another call's result used, on another branch of the session (a native ADK YAML app's agents), is left open: the manager answers those as the stopped turn ends, before it clears the record, the same way (`_answer_unsealed`, shielded; `unsealed`, once ADK's seals are written), so a call that finished keeps its result. `left_open` answers leftovers when `process()` starts (every app: a task-mode agent's delegation, which ADK keeps open on purpose, is refused at build, and ADK's client calls for a human, `adk_request_confirmation`/`_credential`/`_input`, are answered as interrupted, since the manager offers no way to answer them), a result pairing with a call on its own branch only; one a stop left because its answers could not be written is answered `INTERRUPTED_BY_USER` (the manager records the stop). On Deep Agents `InterruptedCallsMiddleware` takes `PatchToolCallsMiddleware`'s place, first in every stack: it records each final `ToolMessage` with the rest of its `Command` update (outermost tool wrapper) and answers open calls in `before_agent`, returning those updates too; the root adds `MARKER` before the new input when the message before it is no reply (a `HumanMessage` or a tool result). Background jobs are not part of the turn: they keep running. Nor is a synchronous tool's function (a sandbox cell, `compile_document`, `execute_python`): it runs in a worker thread on both backends (ADK's tool thread pool, LangChain's executor), which Python cannot stop, so its call is answered at once but the function runs on to its end, keeps its sandbox session busy (the session's next call waits for it) and holds the process's exit until it returns (the sandbox's cleanup stops a cell). So is a knowledge-base ingest's store step (`kb_ingest_text`, `kb_ingest_file`, `ingest_arxiv_paper`: `KnowledgeBase._store` runs `ingest_document` in `asyncio.to_thread`): a cancelled ingest is answered `INTERRUPTED_BY_USER` and still stores its document, so a model that retries it stores a second copy; a cancelled `sidecar` or `backfill_sidecars` keeps its lock or flag until its thread's call ends (`memory/kb/manager.py::_Threads`). +- **Interrupted calls**: Ctrl+C stops the whole turn (loop, model calls, tools, sub-agents), and every call the model made gets exactly one result (`workflow/interrupts.py`, the texts and `OpenCalls`, each manager's record, kept by conversation and by the run that made each call (the root's own, or a sub-agent's: a provider that numbers its call ids per agent gives a sub-agent's call its parent's delegation's id): a call's result as it finished, a call `asking` at its permission prompt, both gates marking it; a conversation whose turn the user cancelled): a finished call keeps its result, a running one or a delegation is `INTERRUPTED_BY_USER`, one at its prompt `NOT_RUN`, one whose job was submitted the job's state (`job_started`); a cancel with no call open leaves `MARKER` as the main agent's reply; a call nothing answered (a crash, an old session) is `INTERRUPTED`. On ADK the turn runs with `abort_signal`, set by `live_events.merged(on_stop=)` before the runner is cancelled, so ADK seals the invocation in its own branches; `merged` waits for its source to close, a repeated cancel too (up to `STOP_GRACE`), and `process()`/`resume_with_job_result()` close their inner generators with `aclosing`, so a run never outlives its turn. `adk/interrupted_calls_plugin.py::InterruptedCallsPlugin` (before `ToolResultCapPlugin`, every app) sets the active tool call around each call (`before_tool_callback`, so a question marks it `asking`), records results with their `state_delta`/`artifact_delta` in `after_tool_callback` (not a long-running starter's `None`: its job answers it), keeps the cap's cut of one (`ToolResultCapPlugin.on_cut`, so a seal sends that cut and its full output is saved once) and rewrites the seals (`error_code` `INVOCATION_ABORTED`, contract-tested) in `on_event_callback`, carrying those deltas, each seal answered from the record of the active run (a sub-agent run's runner stops, and seals its child session, in the run's own task). ADK's seals pair a result with a call by id alone, so a call whose id another call's result used, on another branch of the session (a native ADK YAML app's agents), is left open: the manager answers those as the stopped turn ends, before it clears the record, the same way (`_answer_unsealed`, shielded; `unsealed`, once ADK's seals are written), so a call that finished keeps its result. `left_open` answers leftovers when `process()` starts (every app: a task-mode agent's delegation, which ADK keeps open on purpose, is refused at build, and ADK's client calls for a human, `adk_request_confirmation`/`_credential`/`_input`, are answered as interrupted, since the manager offers no way to answer them), a result pairing with a call on its own branch only; one a stop left because its answers could not be written is answered `INTERRUPTED_BY_USER` (the manager records the stop). A turn that fails (its one `ERROR`) keeps the record of its calls that returned (`OpenCalls.failed`), on both backends: ADK stores a step's parallel results together, after the last, and Deep Agents' graph drops a failed step's writes, so a call that returned beside one the failure stopped is answered at the next turn with its result and the state it changed (ADK: `left_open`, the root's own calls, the record forgotten once its answers are stored), any other `INTERRUPTED`. On Deep Agents `InterruptedCallsMiddleware` takes `PatchToolCallsMiddleware`'s place, first in every stack: it records each final `ToolMessage` with the rest of its `Command` update (outermost tool wrapper) and answers open calls in `before_agent`, returning those updates too; the root adds `MARKER` before the new input when the message before it is no reply (a `HumanMessage` or a tool result). Background jobs are not part of the turn: they keep running. Nor is a synchronous tool's function (a sandbox cell, `compile_document`, `execute_python`): it runs in a worker thread on both backends (ADK's tool thread pool, LangChain's executor), which Python cannot stop, so its call is answered at once but the function runs on to its end, keeps its sandbox session busy (the session's next call waits for it) and holds the process's exit until it returns (the sandbox's cleanup stops a cell). So is a knowledge-base ingest's store step (`kb_ingest_text`, `kb_ingest_file`, `ingest_arxiv_paper`: `KnowledgeBase._store` runs `ingest_document` in `asyncio.to_thread`): a cancelled ingest is answered `INTERRUPTED_BY_USER` and still stores its document, so a model that retries it stores a second copy; a cancelled `sidecar` or `backfill_sidecars` keeps its lock or flag until its thread's call ends (`memory/kb/manager.py::_Threads`). - **Session identity**: durable sessions are addressed by `SessionRef(app_name, user_id, session_id)` (`workflow/sessions.py`). Every session hook (`session_exists`/`list_sessions`/`session_info`/`delete_session`/`recent_messages`/`load_session`) takes an optional `user_id`, defaulting to `settings.default_user` only when the caller omits it. `list_sessions(limit=)` returns `SessionInfo` rows, newest first, and `session_info` one conversation's or None: the title is the first prompt (`session_title`), the turns what the user typed (a job's notice or resume, a delegated request and a summary are none), `updated` when it last changed. ADK keeps the title and the turns in two session-state keys saved with each user turn (`workflow/adk/sessions.py`, `run_async(state_delta=)`; a conversation from before them is read from its events and never written back, which would move it in the list), Deep Agents keeps them in a session index, a LangGraph store beside the checkpointer (`workflow/deepagents/sessions.py`: each row written as a turn starts, its input counted, then from the newest root checkpoint as it ends and after `/compact`, so a listing reads no checkpoint; a database from before the index scanned once per app and user), LangGraph knows only the id; ADK's `delete_session` looks the session up first, so an unknown id is False. ADK keeps each sub-agent run's transcript in a child session of its conversation (`adk/sessions.py`: app `child_app(app_name)`, `--agents`, id `/`, the conversation's user), under an app no listing reads, so none lists or opens as a conversation (`list_sessions`, `session_info`, `session_exists`, a reader's, the CLI's look, `--continue`; `recent_messages` reads the conversation's own session); `ServiceSessions.delete` (the manager's `delete_session` and a reader's, so `/sessions --delete` and the memory store's `/new` too) deletes them, events and all, after the conversation's own delete and whatever that found (`delete_children`: `/` and no further `/`). Instances are numbered per manager, so a manager numbers a conversation's past the ones its store keeps (`_stored_instances`, the highest number of each type, read from the child session ids; `Delegations` asks it once per conversation per turn, before the turn's first run, and again after an ask that failed: another process may have run the conversation between two turns; `AgentRuns.number_from`, which never lowers a count): after a restart, a switch of backend or store, or another process's turns, the next run is never a second `researcher-1` (a model switch keeps the manager, and its numbering), and a stored transcript is never replaced (ADK's `create_session` refuses its id: the run fails). Deep Agents stores no sub-agent transcript, so its manager keeps the base's none. Backends without a session store leave `supports_sessions` False, and the base hooks raise `NotImplementedError` rather than answering with a misleading `False`/`[]`. Deep Agents keys each conversation's checkpointer thread by the whole triple (`workflow/deepagents/sessions.py::thread_id`, `json.dumps([app_name, user_id, session_id])`) and stores it in each checkpoint's metadata, so the backfill's scan filters by app and user and keeps the root namespace; sub-agents are compiled with `checkpointer=False` and store none (an older version's sit under `tools:`, and `delete_session` removes them with the thread and the row, returning False for an unknown id). A backend's conversations are read without its manager through its `SessionReader` (`workflow/session_readers.py::open_session_reader`: the backend's own session code over its store, opened from the settings), so the CLI lists, opens and deletes the conversations of every installed backend (`cli/stored_conversations.py`): each row names its backend (`SessionInfo.backend`), and a conversation another backend keeps opens there (no index joins the stores, and an id two backends keep is two conversations). - **Active turn**: the in-flight `(user, session)` is a `ContextVar` (`workflow/sessions.py::get_active_turn`), set with a token by `_workflow_context()`, which every turn enters through `_turn()` — concurrent turns on one manager stay isolated and nesting restores the outer turn. The tool call in progress is another (`get_active_tool_call()`, a `ToolCallRef(call_id, name)`), set by Deep Agents' `ActiveToolCallMiddleware`, innermost in the tool stack (after the gate and the error middleware, so only a call the gate let through runs as one); on ADK, `InterruptedCallsPlugin.before_tool_callback` sets it around each call (ADK's stand-in for an unknown tool name excepted) and its `after_tool_callback` resets it. The sub-agent run in progress is a third (`get_active_run()`, a `RunRef`; None in the root's own calls), set by `Delegations.delegate` in the run's own task (`workflow/agents/delegate.py`), on both backends, where the run's runner or graph runs (with no tool call active there): a sub-agent's model calls, tool calls, permission checks and questions see it. `JobManager.submit(resume_on_complete=True)` takes `call_id` from the active tool call when the tool passed none, and `call_name` when that is None too; every `submit` records the sub-agent run in progress as the job's `started_by` (`instance_of(get_active_run())`). - **Job notifications**: both backends tell a finished job through the base's `resume_with_job_result` (a template: the record names its conversation, a turn of it (`_turn`), the conversation there (`_conversation_exists`), the job's result read on a worker thread (`JobManager.result` takes the jobs directory's cross-process lock), the notice `job_notice`, told by the backend's `_deliver_job_notice`; `can_resume` makes the same checks without the turn), and the CLI resumes where `supports_job_resume` and `can_resume` say so. On ADK a finished job is told to the conversation (the root, or a native app's agent the root transferred it to) as one user-role text, `tools/jobs/resume.py::job_notice` (the Deep Agents notice below), whoever started it: its call already has a result (the root's "pending", or a sub-agent's job id), and a second result for the call itself made ADK's rearranger send the call, that result and nothing between, so the root answered without the turns since. A native ADK YAML app's job is told the same way (it used to answer its pending call as a `FunctionResponse`, and needed `record.call_id`); a call still open (a starter that returned nothing) is answered first (`_answer_left_open`). A sub-agent's long-running tools are not wrapped (`_build_agent` wraps the root's only): ADK pauses the whole invocation on a pending long-running call, inside a delegation too, so a sub-agent's starter returns the job id as an ordinary result. ADK's `recent_messages` leaves the notice out by its header, since ADK keeps no mark on a message. The CLI delivers only the jobs that wait for the current conversation, by one rule (`cli/waiting_jobs.py`) for the deliveries, the status bar's `↻N` and the line startup says: a finished job flagged `resume_on_complete` whose delivery is still `PENDING` (`JobRecord.awaiting_resume`: not one being delivered), that is this conversation's, or has none, or whose conversation is gone, `can_resume` saying so and no other installed backend's store keeping it (`waiting_jobs.opens_anywhere`, `BaseCLIApp._kept_elsewhere`) (those two told it is unavailable): another conversation's waits until it is opened, `/resume` (`resume_session`) telling them when `job_auto_resume` is on, startup only naming them (`_name_its_waiting_jobs`: outside the input handler Ctrl+C cannot stop a turn); the status bar's `↻N` counts them whatever the setting (`↻N: /jobs resume` when it is off; its `can_resume` answers kept 30 s, forgotten after `/sessions --delete`, a conversation `/new` drops on the memory store, or a new conversation on another backend). On Deep Agents a finished job's result cannot be the pending call's second result (the call already has its `pending` one, and Anthropic and OpenAI take one result per call), so `resume_with_job_result` runs a turn whose input is one `HumanMessage`: `tools/jobs/resume.py::JOB_NOTICE_HEADER`, the ask (`job_notice_ask`: for the root's own job `JOB_NOTICE_ASK`, tell the user what it means: without it a model may say nothing; for a sub-agent's, `JOB_NOTICE_ASK_FOR`, the same ask naming the sub-agent), then `job_result_payload(record, result)` as JSON, a line each (`job_notice`), marked `response_metadata={"agentic_cli": {"kind": "job_result", "job_id": ...}}` (`workflow/deepagents/sessions.py::job_notification`). No provider adapter sends that field; `recent_messages`, and so fact extraction, leaves the message out; the checkpoint keeps it for the model. The root agent receives every job, a sub-agent's too (its run ended when its `task` returned), and the notice names the sub-agent whose call started it (`JobRecord.started_by`, its instance, from the active run at submit; `job_notice_ask`: "A background job researcher-2 started has finished…", the root's own `JOB_NOTICE_ASK`). `can_resume` needs the record's session and user and the conversation's thread, not a call id. diff --git a/README.md b/README.md index 9878e72b..bedbc813 100644 --- a/README.md +++ b/README.md @@ -1573,7 +1573,9 @@ is answered "Interrupted by the user before it finished; it may have partly run…", one waiting at its permission prompt or at a question "Not run: the user cancelled it.", and a request stopped before its reply leaves "(Interrupted by the user.)" as the agent's reply. A call a crash left open -is answered the same way on the next turn. Background jobs the request +is answered the same way on the next turn. After a turn that fails (its one +`ERROR`), a call that had returned keeps its result at the next turn, and +any other call the turn left open is answered as interrupted. Background jobs the request started keep running, and a line after the cancel names each still queued or running, with how to stop it: "Still running in the background: build (j1). Stop one with /jobs cancel ." diff --git a/src/agentic_cli/workflow/adk/interrupted_calls_plugin.py b/src/agentic_cli/workflow/adk/interrupted_calls_plugin.py index dd5e1afe..fb055b93 100644 --- a/src/agentic_cli/workflow/adk/interrupted_calls_plugin.py +++ b/src/agentic_cli/workflow/adk/interrupted_calls_plugin.py @@ -20,10 +20,12 @@ keeps its result, one at its prompt was not run. ``left_open`` answers, on the next turn, the calls nothing answered: a -crash's, a cancel's from before this plugin, or a stop's whose answers -could not be written as it stopped. ADK would keep a long-running one -without its result in every later request, which Claude and OpenAI refuse, -and drop any other one, logging "Dropping function calls" on every request. +crash's, a cancel's from before this plugin, a stop's whose answers could +not be written as it stopped, or a failed turn's, whose record keeps the +calls that returned (ADK had not stored their results yet). ADK would keep +a long-running one without its result in every later request, which Claude +and OpenAI refuse, and drop any other one, logging "Dropping function +calls" on every request. Each call is recorded and answered as the run's that made it, the active run (a sub-agent run is the active one in the task its runner runs in, @@ -54,7 +56,6 @@ MARKER, OpenCalls, interrupted, - unrecorded, ) from agentic_cli.workflow.sessions import ( ToolCallRef, @@ -206,14 +207,19 @@ def _invocation(tool_context: Any) -> str | None: return getattr(tool_context, "invocation_id", None) -def _from_record(open_calls: OpenCalls, call_id: str | None, run: str | None, actions: Any) -> Any: - """A stopped turn's call answered from the record, as ``run``'s. +def _from_record( + open_calls: OpenCalls, call_id: str | None, run: str | None, actions: Any, + *, by_user: bool = True, +) -> Any: + """A call left open answered from the record, as ``run``'s. A finished call keeps what the model would have read (``_Finished.read``); the state it changed goes into ``actions``, stored with the answer as - with its result. Any other is answered as ``OpenCalls.answer`` says. + with its result. Any other is answered as ``OpenCalls.answer`` says: + ``by_user`` whether the user stopped its turn (a stopped turn's call, by + default). """ - answer = open_calls.answer(call_id, by_user=True, run=run) + answer = open_calls.answer(call_id, by_user=by_user, run=run) if not isinstance(answer, _Finished): return answer actions.state_delta.update(answer.state_delta) @@ -221,17 +227,6 @@ def _from_record(open_calls: OpenCalls, call_id: str | None, run: str | None, ac return answer.read -def returned(open_calls: OpenCalls, call_id: str | None, run: str | None) -> dict[str, Any] | None: - """What the call ``call_id`` of ``run`` returned, from the record; None if it did not finish. - - As ADK stores the result (text under ``result``), and as the model reads - it (the cap's cut): what the manager ends a delegation's run with when - no event brought its result (``ADKEventProcessor.end_returned_run``). - """ - finished = open_calls.recorded(call_id, run=run) - return _as_response(finished.read) if isinstance(finished, _Finished) else None - - def _as_response(result: Any) -> dict[str, Any]: """A result as ADK stores it: a dict as it is, anything else under ``result``.""" return result if isinstance(result, dict) else {"result": result} @@ -317,47 +312,60 @@ def unsealed(events: list[Event], open_calls: OpenCalls) -> list[Event]: def left_open(events: list[Event], open_calls: OpenCalls) -> list[Event]: - """The events that answer the calls nothing answered: a crash's, or a stop's left over. + """The events that answer the calls nothing answered: a crash's, a failed + turn's, or a stop's left over. A result answers only a call on its own branch: a sub-agent's result - never answers its parent's call of the same id. Each call left open is - answered as no record holds it (``interrupts.unrecorded``: a job's state, - else ``INTERRUPTED_BY_USER`` when the user stopped the conversation's - last turn, ``OpenCalls.take_cancel``, which this asks once, else - ``INTERRUPTED``): every turn clears its conversation's record as it ends - (``OpenCalls.clear``), and a new process has none. Only the root's own - calls (and those of an agent it transferred to) are matched to a job: a - conversation stored while sub-agents ran on delegation branches of its - own session (``@``) may hold a sub-agent's call there, - whose run is numbered no more, so its call is answered interrupted, never - with another run's job of the same id. A stopped turn - answers what ADK's seals missed itself (``unsealed``), so a stop leaves - a call here only when those answers could not be written. The answer is - in the author, branch, scope and invocation of the event that made the - call, so it pairs with its call in that agent's view; calls that share - those are answered in one event, marked as ADK marks a seal. + never answers its parent's call of the same id. The root's own call left + open (and that of an agent it transferred to) is answered from the + record (``OpenCalls.answer``, as the root's). A failed turn keeps there + the calls that returned (``OpenCalls.failed``), whose results ADK had + not stored (it stores a step's parallel results together, after the + last): each is answered with its result and the state it changed. A call + no record holds gets its job's state, else ``INTERRUPTED_BY_USER`` when + the user stopped the conversation's last turn (``OpenCalls.take_cancel``, + which this asks once), else ``INTERRUPTED``: a turn that ends otherwise + clears its conversation's record (``OpenCalls.clear``), and a new + process has none. A conversation stored while sub-agents ran on + delegation branches of its own session (``@``) may hold + a sub-agent's call there, whose run is numbered no more: it is answered + interrupted, never with another run's result or job of the same id. A + stopped turn answers what ADK's seals missed itself (``unsealed``), so a + stop leaves a call here only when those answers could not be written. + The answer is in the author, branch, scope and invocation of the event + that made the call, so it pairs with its call in that agent's view; + calls that share those are answered in one event, marked as ADK marks a + seal. """ by_user = open_calls.take_cancel(get_active_turn()) - return [ - _answer_event(place, [ + answers: list[Event] = [] + for place, calls in _unanswered(events).items(): + actions = EventActions() + answers.append(_answer_event(place, [ types.FunctionResponse( id=call.id, name=call.name, - response=_as_response(_left_open_answer(call.id, place[2], by_user)), + response=_as_response( + _left_open_answer(open_calls, call.id, place[2], by_user, actions) + ), ) for call in calls - ], INTERRUPTED) - for place, calls in _unanswered(events).items() - ] + ], INTERRUPTED, actions)) + return answers -def _left_open_answer(call_id: str | None, branch: str | None, by_user: bool) -> Any: +def _left_open_answer( + open_calls: OpenCalls, call_id: str | None, branch: str | None, by_user: bool, + actions: Any, +) -> Any: """What ``left_open`` answers the call ``call_id`` made on ``branch`` with. - The root's own call (no delegation on its branch) gets its job's state - when it submitted one (``unrecorded``, the root's: ``started_by`` None); - a sub-agent's on a delegation's branch (``@``) cannot be - named any more, so its call gets no job. + The root's own call (no delegation on its branch) is answered from the + record (``_from_record``, as the root's: a call that returned keeps its + result, the state it changed going into ``actions``; one no record holds + gets its job's state when it submitted one, ``started_by`` None); a + sub-agent's on a delegation's branch (``@``) cannot be + named any more, so its call gets neither. """ if any("@" in segment for segment in (branch or "").split(".")): return interrupted(INTERRUPTED_BY_USER if by_user else INTERRUPTED) - return unrecorded(call_id, by_user=by_user) + return _from_record(open_calls, call_id, None, actions, by_user=by_user) diff --git a/src/agentic_cli/workflow/adk/manager.py b/src/agentic_cli/workflow/adk/manager.py index e7167235..d4c82cfa 100644 --- a/src/agentic_cli/workflow/adk/manager.py +++ b/src/agentic_cli/workflow/adk/manager.py @@ -85,7 +85,12 @@ JOB_RESULT, OUTSIDE_CONTENT_KEY, ) -from agentic_cli.workflow.sessions import ConversationChangedError, SessionInfo, get_active_run +from agentic_cli.workflow.sessions import ( + ConversationChangedError, + SessionInfo, + get_active_run, + get_active_turn, +) from agentic_cli.workflow.tool_output import tool_output_dir from agentic_cli.workflow.usage import adk_usage_event @@ -1212,7 +1217,12 @@ async def _answer_left_open(self, session: Session) -> Session: them, and so does a stop whose own answers to the calls ADK's seals missed could not be written (``_answer_unsealed``); ADK would keep a long-running one in every request, which Claude and OpenAI refuse, - and drop any other one with a warning. + and drop any other one with a warning. So does a failed turn, for + the calls whose results it had not stored; its record of those that + returned (``OpenCalls.failed``) answers them with their results. + Once the answers are stored the record is forgotten, so it answers + no later call of the same id (a provider that numbers its call ids + repeats them). ADK keeps two kinds of call open on purpose. A task-mode agent's delegation cannot be built (a native ADK YAML app with one is @@ -1237,6 +1247,7 @@ async def _answer_left_open(self, session: Session) -> Session: logger.debug("left_open_calls_answered", count=sum( len(event.content.parts) for event in answers )) + self._open_calls.clear(get_active_turn()) return session async def _answer_unsealed(self, ref: SessionRef) -> None: @@ -1420,17 +1431,22 @@ async def _run_and_stream( # Every call of the turn is answered now, sealed or returned, # but for one a stop's seals took for answered (they pair by # id alone): it is answered here, from the record, before the - # record is cleared. One a failure left open is answered at - # the next turn (left_open) as interrupted: the state it - # changed, which ADK stores with its result, is gone. + # record is cleared. A failure keeps the record of the calls + # that returned (OpenCalls.failed), as Deep Agents does: ADK + # stores a step's parallel results together, after the last, + # so one that returned beside a call the failure stopped has + # no stored result, and the next turn answers it from the + # record, with the state it changed (left_open). ref = self.session_ref(session_id, user_id) if aborted.is_set(): # Should those answers not be written, left_open gives # them as the user's. self._open_calls.cancelled(ref) await asyncio.shield(self._answer_unsealed(ref)) - else: + elif failure is None: self._open_calls.clear(ref) + else: + self._open_calls.failed(ref) # Drain any remaining LLM events after processing completes if self._llm_logging_plugin: diff --git a/src/agentic_cli/workflow/interrupts.py b/src/agentic_cli/workflow/interrupts.py index 107e0573..480887cb 100644 --- a/src/agentic_cli/workflow/interrupts.py +++ b/src/agentic_cli/workflow/interrupts.py @@ -16,7 +16,11 @@ tool results) leaves ``MARKER`` as the main agent's reply, so the next turn answers the new message rather than resuming the old one; - a call a crash left open (nothing ran on the way out) is answered - ``INTERRUPTED`` on the next turn. + ``INTERRUPTED`` on the next turn; +- a turn that fails (its one ``ERROR``) keeps the record of its calls that + returned (``OpenCalls.failed``): at the next turn each of them that has no + stored result is answered with the result it returned, and any other + call left open ``INTERRUPTED``. Background jobs a long-running tool started are not part of the turn: they go on, and their results arrive later. A call whose job was submitted is @@ -29,9 +33,10 @@ own, or a sub-agent's: ``workflow.sessions.get_active_run``), since a provider that numbers its call ids per agent gives a sub-agent's call the id of its parent's delegation: the backends answer from it (ADK as the -cancelled turn stops, through its ``abort_signal``; Deep Agents before the -first model call of that conversation's next turn, however many turns ran -in others since). +cancelled turn stops, through its ``abort_signal``, and a failed turn's +calls as that conversation's next turn starts; Deep Agents before the first +model call of that conversation's next turn, however many turns ran in +others since). """ from __future__ import annotations @@ -115,11 +120,10 @@ def unrecorded( """The answer for an open call no record holds: its job's state, else why it has none. ``OpenCalls.answer`` gives it for a call it holds nothing for, matching - the job by the instance of the call's run (``started_by``). ADK's - ``left_open`` gives it for the root's own calls it answers (``started_by`` - None), since each turn clears its conversation's record as it ends, and - a new process has none; a sub-agent's call there gets no job, its run - being numbered no more. + the job by the instance of the call's run (``started_by``); ADK's + ``left_open`` answers the root's own calls through it (``started_by`` + None), and a sub-agent's call with no job, its run being numbered no + more. """ record = job_for_call(call_id, started_by=started_by) if record is not None: diff --git a/tests/integration/test_interrupted_calls.py b/tests/integration/test_interrupted_calls.py index 6228f801..4e378ddd 100644 --- a/tests/integration/test_interrupted_calls.py +++ b/tests/integration/test_interrupted_calls.py @@ -937,6 +937,81 @@ async def test_on_deep_agents_a_call_left_open_across_a_restart_is_answered( assert INTERRUPTED in results(claude.last("Answer"))["toolu_w"] +# --- a failed turn ---------------------------------------------------------------------- + + +async def test_on_adk_calls_whose_results_the_store_refused_keep_them(hermetic, monkeypatch): + """The store refuses the step's results (another process changed the + conversation): the turn fails, and the next turn answers each call with + the result it returned, not as interrupted.""" + from google.adk.errors import StaleSessionError + + claude = serve(monkeypatch, Claude(Answer=[ + say("First."), + calls(("ic_fast", {"topic": "X"}, "toolu_a"), ("ic_fast", {"topic": "Y"}, "toolu_b")), + say("Back again."), + ]), "adk") + manager = solo("adk", ic_fast) + try: + await turn(manager, "Hello.") + service = manager._session_service + real = service.append_event + refused: list = [] + + async def stale_once(session, event): + if event.get_function_responses() and not refused: + refused.append(event) + raise StaleSessionError("The session has been modified in storage.") + return await real(session, event) + + monkeypatch.setattr(service, "append_event", stale_once) + failed = await turn(manager, "Look X and Y up.") + await turn(manager, "Hello again.") + finally: + await manager.cleanup() + + assert claude.refused == [] + assert [e.metadata["error_code"] for e in failed if e.type is EventType.ERROR] == [ + "ConversationChangedError", + ] + sent = results(claude.last("Answer")) + assert sent["toolu_a"] == json.dumps({"success": True, "fact": "X is 42"}) + assert sent["toolu_b"] == json.dumps({"success": True, "fact": "Y is 42"}) + + +async def test_on_adk_a_failed_turns_record_answers_no_later_call(hermetic, monkeypatch): + """A failed turn's record answers its own calls left open at the next + turn, and nothing after: a later call of the same id (a provider that + numbers its call ids repeats them), left running by a stop, is the + user's, not the earlier call's result.""" + from agentic_cli.workflow.adk.interrupted_calls_plugin import ABORTED + from agentic_cli.workflow.interrupts import interrupted + + claude = serve(monkeypatch, Claude(Answer=[ + calls(("ic_fast", {"topic": "X"}, "call_0")), + bad_request("Overloaded."), + calls(("ic_wait", {}, "call_0")), + ]), "adk") + manager = solo("adk", ic_fast, ic_wait) + try: + failed = await turn(manager, "Look X up.") + await cancelled(manager, "Wait for it.") + session = await manager._session_service.get_session( + app_name=manager.app_name, user_id="u", session_id="s") + finally: + await manager.cleanup() + + assert claude.refused == [] + assert [e.metadata["recoverable"] for e in failed if e.type is EventType.ERROR] == [False] + stopped = session.events[-1].invocation_id + (answer,) = [ + response.response for event in session.events + if event.invocation_id == stopped and event.error_code == ABORTED + for response in event.get_function_responses() if response.id == "call_0" + ] + assert answer == interrupted(INTERRUPTED_BY_USER) + + # --- the ADK it rests on ---------------------------------------------------------------- diff --git a/tests/workflow/adk/test_interrupted_calls.py b/tests/workflow/adk/test_interrupted_calls.py index 43aed141..cbb01485 100644 --- a/tests/workflow/adk/test_interrupted_calls.py +++ b/tests/workflow/adk/test_interrupted_calls.py @@ -276,3 +276,26 @@ def test_a_sub_agents_call_is_never_answered_with_the_roots_job(self): "researcher@call_9": {"call_0": interrupted(INTERRUPTED)}, "researcher@call_9.leaf@call_1": {"call_0": interrupted(INTERRUPTED)}, } + + async def test_after_a_failed_turn_a_call_that_returned_keeps_its_result(self): + """A failed turn keeps the record of its calls that returned + (``OpenCalls.failed``): the root's call left open that returned is + answered with its result and the state it changed, one that did not + as interrupted.""" + calls = OpenCalls() + plugin = InterruptedCallsPlugin(calls) + events = [_calls("coordinator", None, ("contract_whoami", "c1"), ("read_file", "c2"))] + + with turn_of(CONVERSATION): + await plugin.after_tool_callback( + tool=_tool("contract_whoami"), tool_args={}, + tool_context=_context("c1", state_delta={"seen": True}), + result={"success": True}, + ) + calls.failed(CONVERSATION) + (answer,) = left_open(events, calls) + + assert (answer.author, answer.branch, answer.error_code) == ("coordinator", None, ABORTED) + assert _answers(answer) == {"c1": {"success": True}, "c2": interrupted(INTERRUPTED)} + # Its state change is stored with its answer, as with its result. + assert answer.actions.state_delta == {"seen": True} diff --git a/tests/workflow/test_subagents_contract.py b/tests/workflow/test_subagents_contract.py index 20ecfe3b..d4a4e4b2 100644 --- a/tests/workflow/test_subagents_contract.py +++ b/tests/workflow/test_subagents_contract.py @@ -51,6 +51,7 @@ from agentic_cli.workflow.agents.control import STEP_LIMIT_NOTE, RunControl from agentic_cli.workflow.agents.slots import SlotPool from agentic_cli.workflow.agents.envelope import ( + INTERRUPTED_ERROR, STEP_LIMIT_ERROR, TIME_LIMIT_ERROR, TOKEN_LIMIT_ERROR, @@ -63,8 +64,13 @@ UserInputUnavailable, WorkflowEvent, ) -from agentic_cli.workflow.interrupts import INTERRUPTED_BY_USER, NOT_RUN -from agentic_cli.workflow.permissions import EXEMPT, Capability +from agentic_cli.workflow.interrupts import ( + INTERRUPTED, + INTERRUPTED_BY_USER, + NOT_RUN, + interrupted, +) +from agentic_cli.workflow.permissions import EXEMPT, Capability, PermissionEngine from agentic_cli.workflow.permissions import prompt as permission_prompt from agentic_cli.workflow.permissions.prompt import ALLOW_ONCE_CHOICE, DENY_CHOICE from agentic_cli.workflow.sessions import ( @@ -3160,3 +3166,147 @@ async def test_a_prompt_no_one_can_answer_lends_nothing(self, make_crew, project ("researcher-1", "w1"), ("researcher-2", "w2"), ] assert (crew.slots().free, crew.slots().waiting) == (1, 0) + + +# -- after a failed turn: the next turn reads what each of its calls returned ------------- + +#: Why a permission check fails in ``gate_fails_after``. +GATE_FAILURE = "the permission rules could not be read" + + +def gate_fails_after(monkeypatch, tool: str, condition: Callable[[], bool]) -> None: + """Make the permission check of each ``tool`` call raise once ``condition()`` holds. + + Both gates ask the engine before the tool runs and neither turns an + engine that raises into the call's result, so the turn fails with its + one ``ERROR``; waiting for ``condition`` lets the call's siblings in the + same response return first. + """ + check = PermissionEngine.check + + async def failing(self, tool_name, capabilities, args): + if tool_name == tool: + await until(condition) + raise RuntimeError(GATE_FAILURE) + return await check(self, tool_name, capabilities, args) + + monkeypatch.setattr(PermissionEngine, "check", failing) + + +class TestAfterAFailedTurn: + """A turn that fails ends with its one ``ERROR``. Its calls that returned + keep their results: the next turn's root request carries each one's own, + and the interrupted text only for a call that did not return.""" + + async def test_results_stored_before_a_failed_model_call_are_read_next_turn( + self, make_crew + ): + """Both calls return and their results are stored; the model call + after them fails.""" + crew = make_crew( + [agent("coordinator", tools=(contract_lookup, contract_whoami))], + coordinator=[calls(use("contract_lookup", "c1", q="X"), use("contract_whoami", "c2")), + fails("overloaded"), says("Back again.")], + ) + + events = await crew.turn() + await crew.turn("And now?") + + (error,) = [e for e in events if e.type is EventType.ERROR] + assert "overloaded" in error.content + assert error.metadata["recoverable"] is False + assert crew.requests("coordinator") == 3 + assert crew.result_ids("coordinator") == ["c1", "c2"] + read = crew.results_read("coordinator") + assert {call: json.loads(text) for call, text in read.items()} == { + "c1": {"success": True, "value": 42}, "c2": {"success": True}, + } + + async def test_a_delegation_still_working_when_a_sibling_fails_the_turn_is_interrupted( + self, make_crew, monkeypatch + ): + """``c2``'s permission check fails the turn while ``c1``'s run still + works: the run ends interrupted, and that is what the next turn's + coordinator reads for ``c1``.""" + crew = make_crew( + [agent("coordinator", sub_agents=("researcher",), tools=("read_file",)), + agent("researcher", tools=(contract_hold,))], + coordinator=[calls(delegate("researcher", "Find X.", "c1"), + use("read_file", "c2", path="notes.txt")), + says("Back again.")], + researcher=[calls(use("contract_hold", "r1")), says("X is 42.")], + ) + gate_fails_after(monkeypatch, "read_file", HELD["started"].is_set) + + events = await crew.turn() + await crew.turn("And now?") + + (error,) = [e for e in events if e.type is EventType.ERROR] + assert GATE_FAILURE in error.content + assert [(r.ref, r.status, r.error) for r in crew.records()] == [ + (RESEARCHER, RUN_INTERRUPTED, INTERRUPTED_ERROR), + ] + assert crew.result_ids("coordinator") == ["c1", "c2"] + read = crew.results_read("coordinator") + assert json.loads(read["c1"]) == interrupted(INTERRUPTED) + assert json.loads(read["c2"]) == interrupted(INTERRUPTED) + + async def test_a_call_that_returned_beside_one_that_failed_the_turn_keeps_its_result( + self, make_crew, monkeypatch + ): + """``c2``'s permission check fails the turn once ``c1`` has returned, + before either backend stores ``c1``'s result (ADK stores a step's + parallel results together, after the last; Deep Agents' graph drops + a failed step's writes): the next turn answers it from the record.""" + crew = make_crew( + [agent("coordinator", tools=(contract_lookup, "read_file"))], + coordinator=[calls(use("contract_lookup", "c1", q="X"), + use("read_file", "c2", path="notes.txt")), + says("Back again.")], + ) + gate_fails_after(monkeypatch, "read_file", lambda: crew.recorded("c1")) + + events = await crew.turn() + await crew.turn("And now?") + + (error,) = [e for e in events if e.type is EventType.ERROR] + assert GATE_FAILURE in error.content + assert error.metadata["recoverable"] is False + assert crew.result_ids("coordinator") == ["c1", "c2"] + read = crew.results_read("coordinator") + assert json.loads(read["c1"]) == {"success": True, "value": 42} + assert json.loads(read["c2"]) == interrupted(INTERRUPTED) + + async def test_a_delegation_that_returned_beside_one_that_failed_the_turn_keeps_its_answer( + self, make_crew, monkeypatch + ): + """Its run ends completed with its answer, and the next turn's + coordinator reads that run's envelope, as its record says; the tasks + the run saved reach the conversation's state too.""" + crew = make_crew( + [agent("coordinator", sub_agents=("researcher",), tools=("read_file",), + include_state_tools=True), + agent("researcher", include_state_tools=True)], + coordinator=[calls(delegate("researcher", "Find X.", "c1"), + use("read_file", "c2", path="notes.txt")), + says("Back again.")], + researcher=[calls(use("save_tasks", "r1", tasks=[ + {"description": "find X", "status": "completed"}, + ])), + says("X is 42.")], + ) + gate_fails_after(monkeypatch, "read_file", lambda: crew.recorded("c1")) + + await crew.turn() + await crew.turn("And now?") + + assert [(r.ref, r.status, r.result) for r in crew.records()] == [ + (RESEARCHER, RUN_COMPLETED, "X is 42."), + ] + assert crew.result_ids("coordinator") == ["c1", "c2"] + read = crew.results_read("coordinator") + answer = envelope_read(read["c1"]) + assert (answer["status"], answer["result"]) == (RUN_COMPLETED, "X is 42.") + assert json.loads(read["c2"]) == interrupted(INTERRUPTED) + state = await crew.state() + assert [task["description"] for task in state["tasks"]] == ["find X"]